DataSource.h

Go to the documentation of this file.

#ifndef DATA_SOURCE_H
#define DATA_SOURCE_H

#include <vector>
#include <string>
#include <unordered_map>
#include <set>

using namespace std;

#include <JSONutil.h>
#include "./data_src/EarthquakeUSGS.h"
#include "./data_src/Game.h"
#include "./data_src/Shakespeare.h"
#include "./data_src/GutenbergBook.h"
#include "./data_src/CancerIncidence.h"
#include "./data_src/ActorMovieIMDB.h"
#include "./data_src/Song.h"
#include "./data_src/ElevationData.h"
#include "./data_src/OSMData.h"
#include "./data_src/OSMVertex.h"
#include "./data_src/OSMEdge.h"
#include "./data_src/MovieActorWikidata.h"
#include "./data_src/Amenity.h"
#include "./data_src/Reddit.h"
#include "./data_src/City.h"
#include "ColorGrid.h"
#include "base64.h"
#include <GraphAdjList.h>
#include <ServerComm.h>
#include <Bridges.h>
#include "rapidjson/document.h"
#include "assert.h"
#include "rapidjson/error/en.h"
#include <fstream>
#include <sys/types.h>
#include <sys/stat.h>
#include <stdio.h>


#include <Cache.h>


namespace bridges {
    using namespace bridges::dataset;
    using namespace bridges::datastructure;




    class DataSource {

        private:

            bool debug_flag = false;

            int debug() const {
                return debug_flag;
            }

            void set_debug_flag() {
                debug_flag = true;
            }

            bridges::Bridges* bridges_inst;
            bridges::lruCache my_cache;


            string getOSMBaseURL() const {
                if (sourceType == "local")
                    return "http://localhost:3000/";

                return "http://bridges-data-server-osm.bridgesuncc.org/";
            }
            string getElevationBaseURL() const {
                if (sourceType == "local")
                    return "http://localhost:3000/";

                return "http://bridges-data-server-elevation.bridgesuncc.org/";
            }
            string getGutenbergBaseURL() const {
                if (sourceType == "local")
                    return "http://localhost:3000/";

                if (sourceType == "testing") 
                    return "http://bridges-data-server-gutenberg-t.bridgesuncc.org/";

                return "http://bridges-data-server-gutenberg.bridgesuncc.org/";
            }
            string  getRedditURL() {
                if (sourceType == "testing")
                    return "http://bridges-data-server-reddit-t.bridgesuncc.org";
                else if (sourceType == "local")
                    return "http://localhost:9999";
                else
                    return "http://bridges-data-server-reddit.bridgesuncc.org";
            }

            string sourceType = "live";

            string getUSCitiesURL() {
                return "http://bridgesdata.herokuapp.com/api/us_cities";
            }

        public:
            DataSource(bridges::Bridges* br = nullptr)
                : bridges_inst(br), my_cache(120) {}

            DataSource(bridges::Bridges& br )
                : DataSource(&br) {}

            void setSourceType(string type) {
                if ( !((type == "live") || (type == "testing") || (type == "local")))
                    throw "Incorrect data server type. Must be live, testing or local";
                if ((type == "testing") || (type == "local"))
                    debug_flag = true;

                sourceType = type;
            }

            vector<City> getUSCities (unordered_map<string, string> params) {
                // check the parameters for illegal keys
                // get the legal keys into a set
                set<string> keys;
                keys.insert("city"); keys.insert("state"); keys.insert("country");
                keys.insert("min_elev"); keys.insert("max_elev");
                keys.insert("min_pop"); keys.insert("max_pop");
                keys.insert("min_long"); keys.insert("max_long");
                keys.insert("min_lat"); keys.insert("max_lat");
                keys.insert("limit"); keys.insert("time_zone");
        
                unordered_map<string, string>::iterator it;
                for (it = params.begin(); it != params.end(); it++) {
                    if (keys.find(it->first) == keys.end())
                        throw std::invalid_argument ("\n\nKey value : " + it->first + 
                            " incorrect\n\n Legal key values: \n   'city', 'state', 'country', 'min_lat', 'max_lat', 'min_long', 'max_long', 'min_pop', 'max_pop', 'time_zone', 'limit' ");
                }


                string url = getUSCitiesURL() + "?";
                if (params.find("city") != params.end()) 
                    url += "city=" + params["city"] + "&";
                if (params.find("state") != params.end()) 
                    url += "state=" + params["state"] + "&";
                if (params.find("country") != params.end()) 
                    url += "country=" + params["country"] + "&";
                if (params.find("min_lat") != params.end()) 
                    url += "minLat=" + params["min_lat"] + "&";
                if (params.find("max_lat") != params.end()) 
                    url += "maxLat=" + params["max_lat"] + "&";
                if (params.find("min_long") != params.end()) 
                    url += "minLong=" + params["min_long"] + "&";
                if (params.find("max_long") != params.end()) 
                    url += "maxLong=" + params["max_long"] + "&";
                if (params.find("min_elev") != params.end()) 
                    url += "minElevation=" + params["min_elev"] + "&";
                if (params.find("max_elev") != params.end()) 
                    url += "maxElevation=" + params["max_elev"] + "&";
                if (params.find("min_pop") != params.end()) 
                    url += "minPopulation=" + params["min_pop"] + "&";
                if (params.find("maxPopulation") != params.end()) 
                    url += "max_pop=" + params["max_pop"] + "&";
                if (params.find("limit") != params.end()) 
                    url += "limit=" + params["limit"] + "&";

                // remove the last &
                url = url.substr(0, url.length()-1);

                // make the request
                using namespace rapidjson;
                Document doc;
                doc.Parse(
                    ServerComm::makeRequest(url, {"Accept: application/json"}).c_str()
                );

                // parse the json
                const Value& city_json = doc["data"];
                vector<City> us_cities;
                for (SizeType i = 0; i < city_json.Size(); i++) {
                    const Value& val = city_json[i];
                    us_cities.push_back (
                        City(
                            val["city"].GetString(),
                            val["state"].GetString(),
                            val["country"].GetString(),
                            val["timezone"].GetString(),
                            val["elevation"].GetInt(),
                            val["population"].GetInt(),
                            val["lat"].GetDouble(),
                            val["lon"].GetDouble()
                        ));
                }

                return us_cities;
            }

            vector<Game> getGameData() {
                using namespace rapidjson;
                Document d;

                // request the game dataset and parse it
                d.Parse(
                    ServerComm::makeRequest("http://bridgesdata.herokuapp.com/api/games",
                {"Accept: application/json"}).c_str());

                const Value& D = d["data"];
                vector<Game> wrapper;
                for (SizeType i = 0; i < D.Size(); i++) {
                    const Value& V = D[i];
                    const Value& G = V["genre"];

                    vector<string> genre;
                    for (SizeType j = 0; j < G.Size(); j++) {
                        genre.push_back(G[j].GetString());
                    }
                    wrapper.push_back(
                        Game(   V["game"].GetString(),
                            V["platform"].GetString(),
                            V["rating"].GetDouble(),
                            genre ) );
                }
                return wrapper;
            }
            vector<ActorMovieIMDB> getActorMovieIMDBData(int number = 0) {
                using namespace rapidjson;
                Document d;
                vector<ActorMovieIMDB> wrapper;
                string url = "http://bridgesdata.herokuapp.com/api/imdb?limit=" +
                    to_string(number);

                // retrieve the data and parse
                d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());

                const Value& D = d["data"];
                for (SizeType i = 0; i < D.Size(); i++) {
                    const Value& V = D[i];
                    wrapper.push_back(
                        ActorMovieIMDB(
                            V["actor"].GetString(),
                            V["movie"].GetString()
                        )
                    );
                }
                return wrapper;
            }


            vector<ActorMovieIMDB> getActorMovieIMDBData2() {
                using namespace rapidjson;
                Document d;
                vector<ActorMovieIMDB> wrapper;
                string url = "http://bridgesdata.herokuapp.com/api/imdb2";

                // retrieve the data and parse
                d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());

                const Value& D = d["data"];
                for (SizeType i = 0; i < D.Size(); i++) {
                    const Value& V = D[i];
                    string actor = V["actor"].GetString();
                    string movie = V["movie"].GetString();
                    double rating = V["rating"].GetDouble();
                    const Value& G = V["genres"];
                    vector<string> genres;
                    for (SizeType j = 0; j < G.Size(); j++) {
                        genres.push_back(G[j].GetString());
                    }
                    wrapper.push_back(ActorMovieIMDB( actor, movie, (float)rating, genres));
                }
                return wrapper;
            }

            vector<EarthquakeUSGS> getEarthquakeUSGSData(int number = 0) {
                using namespace rapidjson;
                Document d;
                vector<EarthquakeUSGS> wrapper;
                if (number <= 0) {
                    d.Parse(ServerComm::makeRequest( "http://earthquakes-uncc.herokuapp.com/eq",
                    {"Accept: application/json"}).c_str());
                    for (SizeType i = 0; i < d.Size(); i++) {
                        const Value& V = d[i]["properties"];
                        const Value& G = d[i]["geometry"]["coordinates"];
                        wrapper.push_back(
                            EarthquakeUSGS(
                                V["mag"].GetDouble(),
                                G[0].GetDouble(),
                                G[1].GetDouble(),
                                V["place"].GetString(),
                                V["title"].GetString(),
                                V["url"].GetString(),
                                V["time"].GetString() )
                        );
                    }
                }
                else {
                    d.Parse(ServerComm::makeRequest( "http://earthquakes-uncc.herokuapp.com/eq/latest/" +
                            to_string(number), {"Accept: application/json"}).c_str());

                    const Value& D = d["Earthquakes"];
                    for (SizeType i = 0; i < D.Size(); i++) {
                        const Value& V = D[i]["properties"];
                        const Value& G = D[i]["geometry"]["coordinates"];
                        //              wrapper.push_back({V["mag"].GetDouble(),G[0].GetDouble(),G[1].GetDouble(),V["place"].GetString(),V["title"].GetString(),V["url"].GetString(),V["time"].GetString()});
                        wrapper.push_back(
                            EarthquakeUSGS(
                                V["mag"].GetDouble(),
                                G[0].GetDouble(),
                                G[1].GetDouble(),
                                V["place"].GetString(),
                                V["title"].GetString(),
                                V["url"].GetString(),
                                V["time"].GetString() )
                        );
                    }
                }
                return wrapper;
            }
            vector<Shakespeare> getShakespeareData(string type = "",
                bool textonly = false) {
                using namespace rapidjson;
                Document d;
                vector<Shakespeare> wrapper;

                string url = "http://bridgesdata.herokuapp.com/api/shakespeare/";

                if (type == "plays" || type == "poems")
                    url += "/" + type;
                if (textonly) {
                    url += "?format=simple";
                }
                // retrieve the data and parse
                d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());

                const Value& D = d["data"];
                for (SizeType i = 0; i < D.Size(); i++) {
                    const Value& V = D[i];
                    wrapper.push_back(
                        Shakespeare(
                            V["title"].GetString(),
                            V["type"].GetString(),
                            V["text"].GetString()
                        )
                    );

                }
                return wrapper;
            }
            Song getSong(string songTitle, string artistName = "") {
                using namespace rapidjson;

                Document d;
                vector<Song> wrapper;
                string url = "http://bridgesdata.herokuapp.com/api/songs/find/";
                // retrieve the data and parse
                if (songTitle.size() > 0)
                    url += songTitle;
                else {
                    throw "Incorrect use of getSong. songTitle should be given.";
                }

                if (artistName.size() > 0)
                    url += "?artistName=" + artistName;

                // check for spaces in url and replace them by '%20'
                //TODO should use curl to do that
                string::size_type n = 0;
                while ( (n = url.find(" ", n)) != string::npos) {
                    url.replace(n, 1, "%20");
                    n++;
                }
                if (debug())
                  std::cerr<<"url: "<<url<<"\n";
                d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());

                string artist   = (d.HasMember("artist")) ?
                    d["artist"].GetString() : string();
                string song     = (d.HasMember("song")) ?
                    d["song"].GetString() : string();
                string album    = (d.HasMember("album"))    ?
                    d["album"].GetString() : string();
                string lyrics   = (d.HasMember("lyrics")) ?
                    d["lyrics"].GetString() : string();
                string release_date = (d.HasMember("release_date")) ?
                    d["release_date"].GetString() : string();

                return Song (artist, song, album, lyrics, release_date);
            }
            vector<Song> getSongData() {
                using namespace rapidjson;

                Document d;
                vector<Song> all_songs;

                string url = "http://bridgesdata.herokuapp.com/api/songs/";
                // retrieve the data and parse

                d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());

                const Value& D = d["data"];

                // get the songs and put them into a vector
                for (SizeType i = 0; i < D.Size(); i++) {
                    const Value& v = D[i];

                    //cout << v["artist"].GetString() << endl;
                    string artist   = (v.HasMember("artist")) ? v["artist"].GetString() : string();
                    string song     = (v.HasMember("song")) ? v["song"].GetString() : string();
                    string album    = (v.HasMember("album"))    ? v["album"].GetString() : string();
                    string lyrics   = (v.HasMember("lyrics")) ? v["lyrics"].GetString() : string();
                    string release_date = (v.HasMember("release_date")) ?
                        v["release_date"].GetString() : string();
                    all_songs.push_back( Song ( artist, song, album, lyrics, release_date) );

                }
                return all_songs;
            }
        private:

            GutenbergBook getAGutenbergBookMetaData(const rapidjson::Value& V) {
                using namespace rapidjson;

                const string s_id = V["id"].GetString();

                const int id = std::stoi(s_id);

                string title = V["title"].GetString();

                const Value& A = V["authors"];
                vector<string> authors;
                for (SizeType j = 0; j < A.Size(); j++) 
                    authors.push_back(A[j].GetString());

                const Value& L = V["lang"];
                string lang = L.GetString();

                const Value& da = V["date_added"];
                string data_added = da.GetString();

                const Value& G = V["genres"];
                vector<string> genres;
                for (SizeType j = 0; j < G.Size(); j++) 
                    genres.push_back(G[j].GetString());

                GutenbergBook gbook = GutenbergBook(title, id, authors, lang, genres, data_added);

                return gbook;
            }

        public:
            GutenbergBook getGutenbergBookMetaData(int id = 0) {
                using namespace rapidjson;

                // get the query string to get meta data of book
                string url = getGutenbergBaseURL() + "/meta?id=" + std::to_string(id);

                // make the query
                Document d;
                d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());
                // only 1 book 
                return getAGutenbergBookMetaData(d["book_list"][0]);
            }
            vector<GutenbergBook> getGutenbergBookMetaData(string term, string category) {
                using namespace rapidjson;

                // get the query string to get meta data of book
                string url = getGutenbergBaseURL() + "/search?search=" + 
                            ServerComm::encodeURLPart(term)+ "&type=" 
                            + ServerComm::encodeURLPart(category);
                // make the query
cout << url << endl;
                Document d;
                d.Parse(ServerComm::makeRequest(url, {"Accept: application/json"}).c_str());

                vector<GutenbergBook> book_list;
                int size= d["book_list"].Size();

                book_list.clear();
                for (int k = 0; k < size; k++)
                    book_list.push_back(getAGutenbergBookMetaData(d["book_list"][k]));

                return book_list;
            }

            string getGutenbergBookText(int id = 0)  {
                using namespace rapidjson;

                //URL for data request
                string data_url = getGutenbergBaseURL() + "/book?id=" + std::to_string(id);

                // generate the hash code - use the id
                string hash_value = "gutenberg" + std::to_string(id);

                // get the dataset from cache, else from the server
                string book_data = getDataSetJSON(data_url, hash_value, "gutenberg");
                // parse, and get the book text
                Document d;
                d.Parse(book_data.c_str());

                string s = std::to_string(id);
                return d[s.c_str()].GetString();
            }

            vector<CancerIncidence> getCancerIncidenceData(int num = 0) {
                using namespace rapidjson;

                Document d;
                vector<CancerIncidence> wrapper;
                string url = "http://bridgesdata.herokuapp.com/api/cancer/withlocations";
                if (num > 0) {
                    url += "?limit=" + to_string(num);
                }

                d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());

                // get the JSON dataset
                const Value& D = d["data"];

                CancerIncidence c;
                for (SizeType i = 0; i < D.Size(); i++) {
                    const Value& v = D[i];
                    const Value& age = v["Age"];

                    c.setAgeAdjustedRate( age["Age Adjusted Rate"].GetDouble());
                    c.setAgeAdjustedCI_Lower(age["Age Adjusted CI Lower"].GetDouble());
                    c.setAgeAdjustedCI_Upper(age["Age Adjusted CI Upper"].GetDouble());

                    c.setYear(v["Year"].GetInt());

                    const Value& data = v["Data"];
                    c.setCrudeRate(data["Crude Rate"].GetDouble());
                    c.setCrudeRate_CI_Lower(data["Crude CI Lower"].GetDouble());
                    c.setCrudeRate_CI_Upper(data["Crude CI Upper"].GetDouble());
                    c.setRace(data["Race"].GetString());
                    c.setPopulation(data["Population"].GetInt());
                    c.setEventType(data["Event Type"].GetString());
                    c.setCount(data["Count"].GetInt());

                    c.setAffectedArea(v["Area"].GetString());

                    const Value& loc = v["loc"];
                    c.setLocationX(loc[0].GetDouble());
                    c.setLocationY(loc[1].GetDouble());

                    wrapper.push_back(c);
                }
                return wrapper;
            }
        private:
            OSMData getOSMDataFromJSON (const string& osm_json) {
                using namespace rapidjson;

                Document osm_data;

                osm_data.Parse(osm_json.c_str());

                // create an osm data object
                OSMData osm;

                if (osm_data.HasMember("nodes")) {
                    vector<OSMVertex> vertices;
                    Value& nodes = osm_data["nodes"];

                    vector<long> vertex_ids;
                    // get the vertices
                    for (SizeType i = 0; i < nodes.Size(); i++) {
                        const Value& node = nodes[i];
                        OSMVertex::OSMVertexID id = node[0].GetInt64();

                        vertex_ids.push_back(id);
                        double lat = node[1].GetDouble(), longit = node[2].GetDouble();
                        vertices.push_back(OSMVertex(id, lat, longit));
                    }
                    osm.setVertices(vertices);
                }
                // add vertices to object
                // get the edges

                if (osm_data.HasMember("edges")) {
                    vector<OSMEdge> edges;
                    Value& links = osm_data["edges"];

                    for (SizeType i = 0; i < links.Size(); i++) {
                        const Value& link = links[i];
                        OSMVertex::OSMVertexID id1 = link[0].GetInt64();
                        OSMVertex::OSMVertexID id2 = link[1].GetInt64();
                        double dist = link[2].GetDouble();

                        edges.push_back(OSMEdge(id1, id2, dist));
                    }
                    osm.setEdges(edges);
                }
                // add edges to object

                if (osm_data.HasMember("meta")) {
                    // get lat long range
                    Value& meta = osm_data["meta"];
                    double lat_min = meta["lat_min"].GetDouble();
                    double lat_max = meta["lat_max"].GetDouble();
                    double longit_min = meta["lon_min"].GetDouble();
                    double longit_max = meta["lon_max"].GetDouble();
                    osm.setLatLongRange(lat_min, lat_max, longit_min, longit_max);
                    // get dataset name
                    osm.setName(meta["name"].GetString());
                }
                return osm;
            }

        public:

            OSMData getOSMData (double lat_min, double long_min,
                double lat_max, double long_max, string level = "default") {

                //URL for hash request
                string hash_url = getOSMBaseURL() +
                    "hash?minLon=" + std::to_string(long_min) +
                    "&minLat=" + std::to_string(lat_min) +
                    "&maxLon=" + std::to_string(long_max) +
                    "&maxLat=" + std::to_string(lat_max) +
                    "&level="  + ServerComm::encodeURLPart(level);


                //URL to request map
                string osm_url =
                    getOSMBaseURL() + "coords?minLon=" + std::to_string(long_min) +
                    "&minLat=" + std::to_string(lat_min) +
                    "&maxLon=" + std::to_string(long_max) +
                    "&maxLat=" + std::to_string(lat_max) +
                    "&level="  + ServerComm::encodeURLPart(level);


                // get the data set from the server or, if available, from
                // a local cache
                string osm_json = getDataSetJSON(osm_url, hash_url, "osm");

                //tries to get hash value for bounding box map
                return getOSMDataFromJSON(osm_json);
            }

            vector<Amenity>  getAmenityData(double minLat, double minLon, double
                maxLat, double maxLon, std::string amenity) {

                std::string amenity_url = getOSMBaseURL() + "amenity?minLon=" +
                    ServerComm::encodeURLPart(std::to_string(minLon)) +
                    "&minLat=" + ServerComm::encodeURLPart(std::to_string(minLat)) +
                    "&maxLon=" + ServerComm::encodeURLPart(std::to_string(maxLon)) +
                    "&maxLat=" + ServerComm::encodeURLPart(std::to_string(maxLat)) +
                    "&amenity=" + ServerComm::encodeURLPart(amenity);

                std::string hash_url = getOSMBaseURL() + "hash?minLon=" +
                    ServerComm::encodeURLPart(std::to_string(minLon)) +
                    "&minLat=" + ServerComm::encodeURLPart(std::to_string(minLat)) +
                    "&maxLon=" + ServerComm::encodeURLPart(std::to_string(maxLon)) +
                    "&maxLat=" + ServerComm::encodeURLPart(std::to_string(maxLat)) +
                    "&amenity=" + ServerComm::encodeURLPart(amenity);

                // make the query to the server to get a JSON of the amenities
                // implements caching to keep local copies
                string amenity_json = getDataSetJSON(amenity_url, hash_url, "amenity");

                // parse the data and return amenity objects
                return parseAmenityData (amenity_json);
            }

            vector<Amenity>  getAmenityData(const std::string& location,
                const std::string& amenity) {
                std::string amenity_url = getOSMBaseURL() + "amenity?location=" +
                    ServerComm::encodeURLPart(location) +
                    "&amenity=" + ServerComm::encodeURLPart(amenity);

                std::string hash_url = getOSMBaseURL() + "hash?location=" +
                    ServerComm::encodeURLPart(location) +
                    "&amenity=" + ServerComm::encodeURLPart(amenity);


                // make the query to the server to get a JSON of the amenities
                // implements caching to keep local copies
                string amenity_json = getDataSetJSON(amenity_url, hash_url, "amenity");

                // parse the data and return amenity objects
                return parseAmenityData (amenity_json);
            }

            vector<Amenity> parseAmenityData(string amenity_json) {
                using namespace rapidjson;

                vector<Amenity>  amenities;
                Document amenity_content;

                amenity_content.Parse(amenity_json.c_str());
                if (amenity_content.HasMember("nodes")) {
                    const Value& nodes = amenity_content["nodes"];
                    if (amenity_content.HasMember("meta")) {
                        const Value& meta = amenity_content["meta"];

                        // first get the meta data
                    //  amenities.setCount(meta["count"].GetInt64());
                    //  amenities.setMinLat(meta["minlat"].GetDouble());
                    //  amenities.setMinLon(meta["minlon"].GetDouble());
                    //  amenities.setMaxLat(meta["maxlat"].GetDouble());
                    //  amenities.setMaxLon(meta["maxlon"].GetDouble());

                        Amenity amen;
                        for (SizeType i = 0; i < nodes.Size(); i++) {
                            // get amenity data
                            const Value& node = nodes[i];
                            amen.setId(node[0].GetInt64());
                            amen.setLat(node[1].GetDouble());
                            amen.setLon(node[2].GetDouble());
                            amen.setName(node[3].GetString());

                            // add to the list
                            amenities.push_back(amen);
                        }
                    }
                    else {
                        cout << "meta data not found!\n";
                        throw;
                    }
                }
                else {
                    cout << "nodes data not found!\n";
                    throw;
                }
                return amenities;
            }

            OSMData getOSMData (string location, string level = "default") {
                //URL for hash request
                string hash_url = getOSMBaseURL() + " hash?location=" +
                    ServerComm::encodeURLPart(location) +
                    "&level=" + ServerComm::encodeURLPart(level);

                //URL to request map
                string osm_url = getOSMBaseURL() +
                    "loc?location=" + ServerComm::encodeURLPart(location) +
                    "&level=" + ServerComm::encodeURLPart(level);

                // get the data set from the server or, if available, from
                // a local cache
                string osm_json = getDataSetJSON(osm_url, hash_url, "osm");

                return getOSMDataFromJSON(osm_json);
            }

            bridges::GraphAdjList<int, std::string> getGraphFromAssignment (
                const std::string& user,
                int assignment,
                int subassignment = 0) {

                bridges::GraphAdjList<int, std::string> gr;

                std::string s = this->getAssignment(user, assignment, subassignment);

                rapidjson::Document doc;
                doc.Parse(s.c_str());
                if (doc.HasParseError())
                    throw "Malformed JSON";

                //Access doc["assignmentJSON"]
                const auto& assjson = doc.FindMember("assignmentJSON");

                if (assjson == doc.MemberEnd())
                    throw "Malformed GraphAdjacencyList JSON: no assignmentJSON";

                //Access doc["assignmentJSON"]["data"]
                const auto& dataArray = assjson->value.FindMember("data");

                if (dataArray == assjson->value.MemberEnd()
                    || dataArray->value.IsArray() == false)
                    throw "Malformed GraphAdjacencyList JSON: No data";

                const auto& data = dataArray->value.GetArray()[0];

                //Access doc["assignmentJSON"]["data"][0]["visual"]
                const auto& dataVisual = data.FindMember("visual");

                if (dataVisual == data.MemberEnd() ||
                    dataVisual->value.IsString() == false)
                    throw "Malformed GraphAdjacencyList JSON";

                std::string assignment_type = dataVisual->value.GetString();

                if (assignment_type != "GraphAdjacencyList")
                    throw "Malformed GraphAdjacencyList JSON: Not a GraphAdjacencyList";

                //reconstructing vertices out of nodes, and using the optional "name" as the data associated
                {
                    const auto& nodes = data.FindMember("nodes");
                    if (nodes == data.MemberEnd() ||
                        nodes->value.IsArray() == false)
                        throw "Malformed GraphAdjacencyList JSON: malformed nodes";


                    const auto& nodeArray = nodes->value.GetArray();
                    int nbVertex = nodeArray.Size();
                    for (int i = 0; i < nbVertex; ++i) {
                        std::string name;

                        const auto& vertexJSONstr = nodeArray[i];

                        const auto& nameJSON = vertexJSONstr.FindMember("name");
                        if (nameJSON != vertexJSONstr.MemberEnd()
                            && nameJSON->value.IsString()) {
                            name = nameJSON->value.GetString();
                        }
                        gr.addVertex(i, name);
                    }
                }

                //reconstructing links, and using "label" as data associated with the link
                {
                    const auto& links = data.FindMember("links");
                    if (links == data.MemberEnd() ||
                        links->value.IsArray() == false)
                        throw "Malformed GraphAdjacencyList JSON: malformed links";

                    const auto& linkArray = links->value.GetArray();
                    int nbLink = linkArray.Size();
                    for (int i = 0; i < nbLink; ++i) {
                        std::string name;
                        int src;
                        int dest;
                        int wgt;

                        const auto& linkJSONstr = linkArray[i];

                        //checking label. Maybe does not exist? (is that field optional?)
                        const auto& nameJSON = linkJSONstr.FindMember("label");
                        if (nameJSON != linkJSONstr.MemberEnd()
                            && nameJSON->value.IsString()) {
                            name = nameJSON->value.GetString();
                        }

                        //checking source
                        const auto& srcJSON = linkJSONstr.FindMember("source");
                        if (srcJSON == linkJSONstr.MemberEnd()
                            || srcJSON->value.IsInt() == false) {
                            throw "Malformed GraphAdjacencyList JSON: malformed link";
                        }
                        src = srcJSON->value.GetInt();


                        //checking destination
                        const auto& dstJSON = linkJSONstr.FindMember("target");
                        if (dstJSON == linkJSONstr.MemberEnd()
                            || dstJSON->value.IsInt() == false) {
                            throw "Malformed GraphAdjacencyList JSON: malformed link";
                        }
                        dest = dstJSON->value.GetInt();

                        //checking weight. //why is weight a mandatory parameter?
                        const auto& wgtJSON = linkJSONstr.FindMember("weight");
                        if (wgtJSON == linkJSONstr.MemberEnd()
                            || wgtJSON->value.IsInt() == false) {
                            throw "Malformed GraphAdjacencyList JSON: malformed link";
                        }
                        wgt = wgtJSON->value.GetInt();


                        //adding edge.
                        gr.addEdge(src, dest, name);
                    }
                }

                return gr;
            }

            bridges::ColorGrid getColorGridFromAssignment(const std::string& user,
                int assignment,
                int subassignment = 0) {

                std::string s = this->getAssignment(user, assignment, subassignment);

                rapidjson::Document doc;
                doc.Parse(s.c_str());
                if (doc.HasParseError())
                    throw "Malformed JSON";

                try {
                    std::string assignment_type = doc["assignment_type"].GetString();

                    if (assignment_type != "ColorGrid")
                        throw "Malformed ColorGrid JSON: Not a ColorGrid";
                }
                catch (rapidjson_exception re) {
                    throw "Malformed JSON: Not a Bridges assignment?";
                }


                try {
                    auto& data = doc["data"][0];

                    std::string encoding = data["encoding"].GetString();
                    if (encoding != "RAW" && encoding != "RLE")
                        throw "Malformed ColorGrid JSON: encoding not supported";


                    //Access doc["data"][0]["dimensions"]
                    const auto& dimensions = data["dimensions"];
                    int dimx = dimensions[0].GetInt();
                    int dimy = dimensions[1].GetInt();

                    if (debug())
                        std::cerr << "Dimensions: " << dimx << "x" << dimy << std::endl;

                    //Access doc["data"][0]["nodes"][0]
                    std::string base64_encoded_assignment = data["nodes"][0].GetString();


                    std::vector<bridges::BYTE> decoded = bridges::base64::decode(base64_encoded_assignment);

                    bridges::ColorGrid cg (dimx, dimy);


                    if (encoding == "RAW") {
                        if (debug())
                            std::cerr << "decoding RAW" << std::endl;
                        if (debug())
                            std::cerr << "length: " << decoded.size() << std::endl;
                        if (decoded.size() < dimx * dimy * 4)
                            throw "Malformed ColorGrid JSON: nodes is smaller than expected";

                        //first pixel
                        //std::cerr<<(int)decoded[0]<<" "<<(int)decoded[1]<<" "<<(int)decoded[2]<<" "<<(int)decoded[3]<<std::endl;

                        //bridges::ColorGrid* ptr = new bridges::ColorGrid (dimx, dimy);

                        size_t base = 0;

                        for (int x = 0; x < dimx; ++x) {
                            for (int y = 0; y < dimy; ++y) {
                                bridges::Color c ((int)decoded[base],
                                    (int)decoded[base + 1],
                                    (int)decoded[base + 2],
                                    (int)decoded[base + 3]
                                );

                                cg.set(x, y, c);
                                base += 4;
                            }
                        }
                    }
                    else if (encoding == "RLE") {
                        if (debug())
                            std::cerr << "Decoding RLE" << std::endl;

                        int currentInDecoded = 0;
                        int currentInCG = 0;
                        while (currentInDecoded != decoded.size()) {
                            if (currentInDecoded + 5 > decoded.size())
                                throw "Malformed ColorGrid JSON: nodes is not a multiple of 5";



                            int repeat = (BYTE) decoded[currentInDecoded++];
                            int r = (BYTE) decoded[currentInDecoded++];
                            int g = (BYTE) decoded[currentInDecoded++];
                            int b = (BYTE) decoded[currentInDecoded++];
                            int a = (BYTE) decoded[currentInDecoded++];

                            if (debug())
                                std::cerr << "indecoded: " << currentInDecoded
                                    << " repeat: " << (int)repeat
                                    << " color(" << (int)r << "," << (int)g << "," << (int)b << "," << (int)a << ")"
                                    << std::endl;

                            bridges::Color c (r, g, b, a);

                            while (repeat >= 0) {
                                int posX = currentInCG / dimy;
                                int posY = currentInCG % dimy;
                                if (posX >= dimx || posY >= dimy) {
                                    if (debug())
                                        std::cerr << posX << " " << dimx << " " << posY << " " << dimy << std::endl;
                                    throw "Malformed ColorGrid JSON: Too much data in nodes";
                                }
                                cg.set(posX, posY, c);

                                currentInCG++;
                                repeat --;
                            }
                        }
                        if (debug())
                            std::cerr << "written " << currentInCG << " pixels" << std::endl;
                        if (currentInCG != dimx * dimy)
                            throw "Malformed ColorGrid JSON: Not enough data in nodes";
                    }

                    return cg;
                }
                catch (rapidjson_exception re) {
                    throw "Malformed ColorGrid JSON";
                }

            }
        private:
            /***
             * This function obtains the JSON representation of a particular subassignment.
             *
             * @return a string that is the JSON representation of the subassignment as stored by the Bridges server.
             * @param user the name of the user who uploaded the assignment
             * @param assignment the ID of the assignment to get
             * @param subassignment the ID of the subassignment to get
             ***/
            std::string getAssignment(std::string user,
                int assignment,
                int subassignment = 0) {
                std::vector<std::string> headers;

                std::stringstream ss;

                if (bridges_inst)
                    ss << bridges_inst->getServerURL();
                else
                    ss << bridges::Bridges::getDefaultServerURL();
                ss << "/assignmentJSON/"
                    << assignment << ".";
                ss << std::setfill('0') << std::setw(2) << subassignment;
                ss << "/" << user;

                std::string url = ss.str();


                std::string s = bridges::ServerComm::makeRequest(url, headers);

                return s;
            }


            void removeFirstOccurence (std::string & str, const std::string & toRemove) {
                size_t pos = str.find(toRemove);
                if (pos  != std::string::npos) {
                    str.erase(pos, toRemove.length());
                }
            }

            void  getWikidataActorMovieDirect (int yearbegin, int yearend, std::vector<MovieActorWikidata>& vout) {
                std::string codename = "wikidata-actormovie-" + std::to_string(yearbegin) + "-" + std::to_string(yearend);
                std::string json;
                bool from_cache = false;
                try {
                    if (my_cache.inCache(codename)) {
                        json = my_cache.getDoc(codename);
                        from_cache = true;
                    }
                }
                catch (CacheException& ce) {
                    //something went bad trying to access the cache
                    std::cout << "Exception while reading from cache. Ignoring cache and continue." << std::endl;
                }


                if (!from_cache) {
                    std::vector<std::string> http_headers;
                    http_headers.push_back("User-Agent: bridges-cxx"); //wikidata kicks you out if you don't have a useragent
                    http_headers.push_back("Accept: application/json"); //tell wikidata we are OK with JSON

                    string url = "https://query.wikidata.org/sparql?";

                    //Q1860 is "English"
                    //P364 is "original language of film or TV show"
                    //P161 is "cast member"
                    //P577 is "publication date"
                    //A11424 is "film"
                    //P31 is "instance of"
                    // "instance of film" is necessary to filter out tv shows
                    std::string sparqlquery = 
                        "SELECT ?movie ?movieLabel ?actor ?actorLabel WHERE \
{\
  ?movie wdt:P31 wd:Q11424.\
  ?movie wdt:P161 ?actor.\
  ?movie wdt:P364 wd:Q1860.\
  ?movie wdt:P577 ?date.\
  FILTER(YEAR(?date) >= " + std::to_string(yearbegin) + " && YEAR(?date) <= " + std::to_string(yearend) + ").\
    SERVICE wikibase:label { bd:serviceParam wikibase:language \"en\". } \
}";
                    url += "query=" + ServerComm::encodeURLPart(sparqlquery);
                    url += "&";
                    url += "format=json";


                    if (debug()) {
                      std::cout<<"URL: "<<url<<"\n";
                    }
                    // get the Wikidata json
                    json = ServerComm::makeRequest(url, http_headers);

                    try {
                        my_cache.putDoc(codename, json);
                    }
                    catch (CacheException& ce) {
                        //something went bad trying to access the cache
                        std::cerr << "Exception while storing in cache. Weird but not critical." << std::endl;
                    }
                }

                {
                    using namespace rapidjson;
                    rapidjson::Document doc;
                    doc.Parse(json.c_str());
                    if (doc.HasParseError())
                        throw "Malformed JSON";

                    try {
                        const auto& resultsArray = doc["results"]["bindings"].GetArray();

                        for (auto& mak_json : resultsArray) {
                            MovieActorWikidata mak;

                            // all wikidata uri start with "http://www.wikidata.org/entity/"
                            // so strip it out because it does not help discriminate and
                            // consume memory and runtime to compare string
                            std::string actoruri = mak_json["actor"]["value"].GetString();
                            std::string movieuri = mak_json["movie"]["value"].GetString();
                            removeFirstOccurence (actoruri, "http://www.wikidata.org/entity/");

                            removeFirstOccurence (movieuri, "http://www.wikidata.org/entity/");


                            mak.setActorURI(actoruri);
                            mak.setMovieURI(movieuri);
                            mak.setActorName(mak_json["actorLabel"]["value"].GetString());
                            mak.setMovieName(mak_json["movieLabel"]["value"].GetString());
                            vout.push_back(mak);
                        }

                    }
                    catch (rapidjson_exception re) {
                        throw "Malformed JSON: Not from wikidata?";
                    }
                }
            }
        public:

            std::vector<MovieActorWikidata> getWikidataActorMovie (int yearbegin, int yearend) {
                //Internally this function get the data year by year. This
                //is pretty bad because it hits wikidata the first time
                //for multiple years. But it enables to work around
                //wikidata's time limit.  This also works well because the
                //Cache will store each year independently and so without
                //redundancy.  Though I (Erik) am not completely sure that a
                //movie can be appear in different years, for instance it
                //can be released in the US in 2005 but in canada in
                //2006...

                std::vector<MovieActorWikidata> ret;
                for (int y = yearbegin; y <= yearend; ++y) {
                    cout << "getting year " << y << endl;
                    getWikidataActorMovieDirect (y, y, ret);
                }
                return ret;
            }


            ElevationData getElevationData (
                double minLat, double minLon,
                double maxLat, double maxLon, double res = 0.0166)  {

                // set up the elevation data url to get the data, given
                // a lat/long bounding box

                std::string elev_url = getElevationBaseURL() +
                    "elevation?minLon=" + ServerComm::encodeURLPart(std::to_string(minLon)) +
                    "&minLat=" + ServerComm::encodeURLPart(std::to_string(minLat)) +
                    "&maxLon=" + ServerComm::encodeURLPart(std::to_string(maxLon)) +
                    "&maxLat=" + ServerComm::encodeURLPart(std::to_string(maxLat)) +
                    "&resX=" + ServerComm::encodeURLPart(std::to_string(res)) +
                    "&resY=" + ServerComm::encodeURLPart(std::to_string(res));

                if (debug())
                    cout << "Elevation URL:" << elev_url << "\n";

                std::string hash_url = getElevationBaseURL() +
                    "hash?minLon=" + ServerComm::encodeURLPart(std::to_string(minLon)) +
                    "&minLat=" + ServerComm::encodeURLPart(std::to_string(minLat)) +
                    "&maxLon=" + ServerComm::encodeURLPart(std::to_string(maxLon)) +
                    "&maxLat=" + ServerComm::encodeURLPart(std::to_string(maxLat)) +
                    "&resX=" + ServerComm::encodeURLPart(std::to_string(res)) +
                    "&resY=" + ServerComm::encodeURLPart(std::to_string(res));

                if (debug())
                    cout << "Hash URL:" << hash_url << "\n";


                // get the dataset's JSON from the local cache, if available,
                // else from the server

                string elev_json = getDataSetJSON(elev_url, hash_url, "elevation");

                return parseElevationData(elev_json);
            }

            ElevationData parseElevationData (string elev_json) {

                // use a string stream to parse the data, which is not really a JSON,
                // but raw text
                stringstream ss(elev_json);

                int rows, cols, elev_val;
                double ll_x, ll_y, cell_size;
                string tmp;

                // get the dimensions, origin
                ss >> tmp >> cols >> tmp >> rows >>
                    tmp >> ll_x >> tmp >> ll_y >>
                    tmp >> cell_size;


                // create the elevation object
                ElevationData elev_data (rows, cols);
                elev_data.setxll(ll_x);
                elev_data.setyll(ll_y);
                elev_data.setCellSize(cell_size);

                // load the elevation data
                for (int i = 0; i < rows; i++) {
                    for (int j = 0; j < cols; j++) {
                        ss >> elev_val;
                        elev_data.setVal(i, j, elev_val);
                    }
                }
                return elev_data;
            }

      std::vector<std::string> getAvailableSubreddits() {
            string base_url = getRedditURL();
            string url = base_url + "/listJSON";
              if (debug()) {
                std::cout<<"hitting url: "<<url<<"\n";
              }         
            using namespace rapidjson;
            Document doc;
            {
              std::string s = ServerComm::makeRequest(url, {"Accept: application/json"});
              if (debug()) {
                std::cout<<"Returned JSON:"<<s<<"\n";
              }
              try {
                doc.Parse(s.c_str());
              } catch(rapidjson_exception& re) {
                std::cerr<<"malformed subreddit list"<<"\n";
                std::cerr<<"Original exception: "<<(std::string)re<<"\n";
              }
            }

            std::vector<std::string> subreddits;
              try {
                for (auto& m : doc.GetArray()) {

                  std::string subred = m.GetString();
                  subreddits.push_back(subred);

                }
              } catch(rapidjson_exception& re) {
                std::cerr<<"malformed subreddit list"<<"\n";
                std::cerr<<"Original exception: "<<(std::string)re<<"\n";
              }

            return subreddits;
            
      }

            vector<Reddit> getRedditData(string subreddit, int time_request=-9999) {
                string base_url = getRedditURL();
                if (debug()) {
                  cout <<  "reddit base url:" << base_url <<  "\n";
                }
                string url = base_url + "/cache?subreddit=" + subreddit + 
                    "&time_request=" + std::to_string(time_request);

                if (debug()) {
                  cout<<  "reddit url:" << url <<  "\n";
                }

                
                using namespace rapidjson;
                Document doc;
                {
                  std::string s = ServerComm::makeRequest(url, {"Accept: application/json"});
                  if (debug()) {
                    std::cout<<"Returned JSON:"<<s<<"\n";
                  }
                  doc.Parse(s.c_str());
                }
                
                vector<Reddit> reddit_posts;
                for (auto& m : doc.GetObject()) {
                  try {
                    if (debug()) {
                      std::cout<<m.name.GetString()<<"\n";
                    }
                    auto& postJSON = m.value;
                    
                    std::string id = postJSON["id"].GetString();
                    std::string title = postJSON["title"].GetString();
                    std::string author = postJSON["author"].GetString();
                    int score = postJSON["score"].GetInt();
                    float vote_ratio = postJSON["vote_ratio"].GetDouble();
                    int comment_count = postJSON["comment_count"].GetInt();
                    std::string subreddit = postJSON["subreddit"].GetString();
                    int posttime = postJSON["post_time"].GetDouble();
                    std::string url = postJSON["url"].GetString();
                    std::string text = postJSON["text"].GetString();
                    
                    
                    Reddit r;
                    r.setID(id);
                    r.setTitle(title);
                    r.setAuthor(author);
                    r.setScore(score);
                    r.setVoteRatio(vote_ratio);
                    r.setCommentCount(comment_count);
                    r.setSubreddit(subreddit);
                    r.setPostTime(posttime);
                    r.setURL(url);
                    r.setText(text);
                    reddit_posts.push_back(r);
                  }
                  catch(rapidjson_exception& re) {
                    std::cerr<<"malformed Reddit post"<<"\n";
                    std::cerr<<"Original exception: "<<(std::string)re<<"\n";
                  }
                }


                return reddit_posts;
            }

        private:
            string getHashCode (string hash_url, string data_type) {
                string hash_value; 
                if (data_type == "osm" || data_type == "amenity" || 
                        data_type == "elevation") {
                    hash_value = ServerComm::makeRequest(hash_url, {"Accept: application/json"});
                }
                else if (data_type == "gutenberg") 
                    hash_value = hash_url; 
            
                return hash_value;
            }

            std::string getDataSetJSON(std::string data_url, std::string hash_url, 
                                                std::string data_type) {

                std::string data_json = "";

                // First check to see if the requested data is stored in local cache
                // get hash value for elevation data
                if (debug())
                    cerr << "Checking the cache: Hash url: " << hash_url << "\n";

                // generate the hash code
                string hash_value = getHashCode(hash_url, data_type);

                bool dataloaded=false;
                
                if ((hash_value != "false") && (my_cache.inCache(hash_value) == true)) { //local cache contains the dataset
                    try {
                        data_json = my_cache.getDoc(hash_value);
                        dataloaded = true;
                    }
                    catch (CacheException& ce) {
                        //something went bad trying to access the data in the local cache
                        std::cout << "Exception while reading from cache. "
                            << "Ignoring cache and continuing..\n.";
                    }
                }
                if (!dataloaded) {
                  //Data could not get accessed from cache for some reason.
                  //So teh data need to be access from the remote server.
                  //Then we will store it in a local cache for future usage.
                    if (debug())
                        std::cerr << "Hitting data URL: " << data_url << "\n";

                    //Requests the data
                    data_json = ServerComm::makeRequest(data_url,
                                    {"Accept: application/json"});

                    //Store the data in cache for future reuse
                    try {
                      // We need the data's hash code to know where to store it in local cache.
                      // We may already have it from the previous query.
                      if (hash_value == "false") {
                        if (debug())
                          std::cerr << "Hitting hash URL: " << hash_value << "\n";

                        hash_value = getHashCode(hash_url, data_type);
                      }

                      // This test should only ever be true if something wrong happens server-side
                      if (hash_value == "false") {
                        std::cerr << "Error while gathering hash value for "<<data_type<<" dataset..\n"
                              << "Weird but not critical.\n";
                      }
                      else {
                        my_cache.putDoc(hash_value, data_json);
                      }
                    }
                    catch (CacheException& ce) {
                        //something went bad trying to access the cache
                        std::cerr << "Exception while storing in cache. " <<
                            "Weird but not critical.\n";
                        if (debug())
                            std::cerr << "Tried to store hash=" << hash_value <<
                                " key = " << data_json << std::endl;
                    }
                }

                if (debug())
                    cout << data_json;

                return data_json;
            }

    }; // class DataSource
} // namespace bridges
#endif