DataSource.h

Go to the documentation of this file.

#ifndef DATA_SOURCE_H
#define DATA_SOURCE_H
 
#include <vector>
#include <string>
#include <unordered_map>
#include <set>
 
using namespace std;
 
#include <JSONutil.h>
#include "./data_src/EarthquakeUSGS.h"
#include "./data_src/Game.h"
#include "./data_src/Shakespeare.h"
#include "./data_src/GutenbergBook.h"
#include "./data_src/CancerIncidence.h"
#include "./data_src/ActorMovieIMDB.h"
#include "./data_src/Song.h"
#include "./data_src/ElevationData.h"
#include "./data_src/OSMData.h"
#include "./data_src/OSMVertex.h"
#include "./data_src/OSMEdge.h"
#include "./data_src/MovieActorWikidata.h"
#include "./data_src/Amenity.h"
#include "./data_src/Reddit.h"
#include "./data_src/City.h"
#include "./data_src/USState.h"
#include "./data_src/USCounty.h"
#include "ColorGrid.h"
#include "base64.h"
#include <GraphAdjList.h>
#include <ServerComm.h>
#include <Bridges.h>
#include "rapidjson/document.h"
#include "assert.h"
#include "rapidjson/error/en.h"
#include <fstream>
#include <sys/types.h>
#include <sys/stat.h>
#include <stdio.h>
 
#include <Cache.h>
 
namespace bridges {
    using namespace bridges::dataset;
    using namespace bridges::datastructure;
 
    class DataSource {
 
        private:
 
            bool debug_flag = false;
 
            int debug() const {
                return debug_flag;
            }
 
            void set_debug_flag() {
                debug_flag = true;
            }
 
            bridges::Bridges* bridges_inst;
            bridges::lruCache my_cache;
 
            string getOSMBaseURL() const {
                if (sourceType == "local")
                    return "http://localhost:3000/";
 
                return "http://bridges-data-server-osm.bridgesuncc.org/";
            }
            string getElevationBaseURL() const {
                if (sourceType == "local")
                    return "http://localhost:3000/";
 
                return "http://bridges-data-server-elevation.bridgesuncc.org/";
            }
            string getGutenbergBaseURL() const {
                if (sourceType == "local")
                    return "http://localhost:3000/";
 
                if (sourceType == "testing")
                    return "http://bridges-data-server-gutenberg-t.bridgesuncc.org/";
 
                return "http://bridges-data-server-gutenberg.bridgesuncc.org/";
            }
            string  getRedditURL() {
                if (sourceType == "testing")
                    return "http://bridges-data-server-reddit-t.bridgesuncc.org";
                else if (sourceType == "local")
                    return "http://localhost:9999";
                else
                    return "http://bridges-data-server-reddit.bridgesuncc.org";
            }
 
            string sourceType = "live";
 
            string getUSCitiesURL() {
                return "http://bridgesdata.herokuapp.com/api/us_cities";
            }
 
            string getWorldCitiesURL() {
                return "http://bridgesdata.herokuapp.com/api/world_cities";
            }
 
            string getUSStateCountiesURL() {
                return "http://bridgesdata.herokuapp.com/api/us_map?state=";
            }
 
            void defaultDebug() {
                char* force = getenv("FORCE_BRIDGES_DATADEBUG");
                if (force != nullptr)
                    set_debug_flag();
 
            }
 
        public:
            DataSource(bridges::Bridges* br = nullptr)
                : bridges_inst(br), my_cache(120) {
                defaultDebug();
            }
 
            DataSource(bridges::Bridges& br )
                : DataSource(&br) {
                defaultDebug();
            }
 
            void setSourceType(string type) {
                if ( !((type == "live") || (type == "testing") || (type == "local")))
                    throw "Incorrect data server type. Must be live, testing or local";
                if ((type == "testing") || (type == "local"))
                    debug_flag = true;
 
                sourceType = type;
            }
 
            vector<City> getUSCities (unordered_map<string, string> params) {
                // check the parameters for illegal keys
                // get the legal keys into a set
                set<string> keys;
                keys.insert("city");
                keys.insert("state");
                keys.insert("country");
                keys.insert("min_elev");
                keys.insert("max_elev");
                keys.insert("min_pop");
                keys.insert("max_pop");
                keys.insert("min_long");
                keys.insert("max_long");
                keys.insert("min_lat");
                keys.insert("max_lat");
                keys.insert("limit");
                keys.insert("time_zone");
 
                unordered_map<string, string>::iterator it;
                for (it = params.begin(); it != params.end(); it++) {
                    if (keys.find(it->first) == keys.end())
                        throw std::invalid_argument ("\n\nKey value : " + it->first +
                            " incorrect\n\n Legal key values: \n   'city', 'state', 'country', 'min_lat', 'max_lat', 'min_long', 'max_long', 'min_pop', 'max_pop', 'time_zone', 'limit' ");
                }
 
                string url = getUSCitiesURL() + "?";
                if (params.find("city") != params.end())
                    url += "city=" + params["city"] + "&";
                if (params.find("state") != params.end())
                    url += "state=" + params["state"] + "&";
                if (params.find("country") != params.end())
                    url += "country=" + params["country"] + "&";
                if (params.find("min_lat") != params.end())
                    url += "minLat=" + params["min_lat"] + "&";
                if (params.find("max_lat") != params.end())
                    url += "maxLat=" + params["max_lat"] + "&";
                if (params.find("min_long") != params.end())
                    url += "minLong=" + params["min_long"] + "&";
                if (params.find("max_long") != params.end())
                    url += "maxLong=" + params["max_long"] + "&";
                if (params.find("min_elev") != params.end())
                    url += "minElevation=" + params["min_elev"] + "&";
                if (params.find("max_elev") != params.end())
                    url += "maxElevation=" + params["max_elev"] + "&";
                if (params.find("min_pop") != params.end())
                    url += "minPopulation=" + params["min_pop"] + "&";
                if (params.find("maxPopulation") != params.end())
                    url += "max_pop=" + params["max_pop"] + "&";
                if (params.find("limit") != params.end())
                    url += "limit=" + params["limit"] + "&";
 
                // remove the last &
                url = url.substr(0, url.length() - 1);
 
                // make the request
                using namespace rapidjson;
                Document doc;
                doc.Parse(
                    ServerComm::makeRequest(url, {"Accept: application/json"}).c_str()
                );
 
                // parse the json
                const Value& city_json = doc["data"];
                vector<City> us_cities;
                for (SizeType i = 0; i < city_json.Size(); i++) {
                    const Value& val = city_json[i];
                    us_cities.push_back (
                        City(
                            val["city"].GetString(),
                            val["state"].GetString(),
                            val["country"].GetString(),
                            val["timezone"].GetString(),
                            val["elevation"].GetInt(),
                            val["population"].GetInt(),
                            val["lat"].GetDouble(),
                            val["lon"].GetDouble()
                        ));
                }
 
                return us_cities;
            }
            /*
                        vector<City> getWorldCities (unordered_map<string, string> params) {
                            string url = getWorldCitiesURL() + "?";
                            if (params.find("city") != params.end())
                                url += "city=" + params["city"] + "&";
                            if (params.find("state") != params.end())
                                url += "state=" + params["state"] + "&";
                            if (params.find("country") != params.end())
                                url += "country=" + params["country"] + "&";
                            if (params.find("min_pop") != params.end())
                                url += "minPopulation=" + params["min_pop"] + "&";
                            if (params.find("maxPopulation") != params.end())
                                url += "max_pop=" + params["max_pop"] + "&";
                            if (params.find("min_lat_long") != params.end())
                                url += "minLatLong=" + params["minll"][0] + "," +
                                                    params["minll"][1] + "&";
                            if (params.find("max_lat_long") != params.end())
                                url += "maxLatLong=" + params["maxll"][0] + "," +
                                                    params["maxll"][1] + "&";
                            if (params.find("min_elev") != params.end())
                                url += "minElevation=" + params["min_elev"] + "&";
                            if (params.find("max_elev") != params.end())
                                url += "maxElevation=" + params["max_elev"] + "&";
                            if (params.find("limit") != params.end())
                                url += "limit=" + params["limit"] + "&";
 
                            // remove the last &
                            url = url.substr(0, url.length() - 1);
 
                            // make the request
                            using namespace rapidjson;
                            Document doc;
                            doc.Parse(
                                ServerComm::makeRequest(url, {"Accept: application/json"}).c_str()
                            );
                            // parse the json
                            const Value& city_json = doc["data"];
                            vector<City> world_cities;
                            for (SizeType i = 0; i < city_json.Size(); i++) {
                                const Value& val = city_json[i];
                                _cities.push_back (
                                    City(
                                        val["city"].GetString(),
                                        val["state"].GetString(),
                                        val["country"].GetString(),
                                        val["timezone"].GetString(),
                                        val["elevation"].GetInt(),
                                        val["population"].GetInt(),
                                        val["lat"].GetDouble(),
                                        val["lon"].GetDouble()
                                    ));
                            }
                            return world_cities;
                        }
            */
 
            // list of all states
            const vector<string> all_states = {"Alabama", "Alaska", "Arizona", "Arkansas", "California", "Colorado", "Connecticut", "Delaware", "Florida", "Georgia", "Hawaii", "Idaho", "Illinois", "Indiana", "Iowa", "Kansas", "Kentucky", "Louisiana", "Maine", "Maryland", "Massachusetts", "Michigan", "Minnesota", "Mississippi", "Missouri", "Montana", "Nebraska", "Nevada", "New Hampshire", "New Jersey", "New Mexico", "New York", "North Carolina", "North Dakota", "Ohio", "Oklahoma", "Oregon", "Pennsylvania", "Rhode Island", "South Carolina", "South Dakota", "Tennessee", "Texas", "Utah", "Vermont", "Virginia", "Washington", "West Virginia", "Wisconsin", "Wyoming"};
 
            vector<USState> getUSMapData () {
                return getUSMapCountyData(all_states, false);
            }
 
            vector<USState> getUSMapCountyData () {
                return getUSMapCountyData(all_states, true);
            }
 
            vector<USState> getUSMapCountyData (vector<string> state_names,
                bool view_counties = true) {
 
                // form the http query
                string url = getUSStateCountiesURL();
                for (auto& k : state_names)
                    url += ServerComm::encodeURLPart(k) + ',';
 
                // remove the last comma
                url = url.substr(0, url.size() - 1);
 
                if (debug())
                    std::cerr << "Hitting: " << url << std::endl;
 
                // make the request
                using namespace rapidjson;
                Document doc;
                doc.Parse(
                    ServerComm::makeRequest(url, {"Accept: application/json"}).c_str()
                );
 
                // iterate through the states to parse and store in
                // the State and County objects
                vector<USState> states;
                const Value& state_data =  doc["data"];
                for (SizeType i  = 0; i < state_names.size(); i++) {
                    const Value& st = state_data[i];
                    const Value& county_data = st["counties"];
                    const Value& st_name = st["_id"]["input"];
 
                    // create the state object and add it to a list
                    states.push_back(USState(st_name.GetString()));
                    unordered_map<string, USCounty> counties = states[i].getCounties();
 
                    states[i].setViewCountiesFlag(view_counties);
 
                    // get county data
                    if (view_counties) {
                        for (SizeType j = 0; j < county_data.Size(); j++) {
                            const Value& val = county_data[j];
                            // get its geoid
                            string geoid = (val["properties"]["GEOID"]).GetString();
                            counties[geoid] = USCounty(geoid,
                                    (val["properties"]["FIPS_CODE"]).GetString(),
                                    (val["properties"]["COUNTY_STATE_CODE"]).GetString(),
                                    (val["properties"]["COUNTY_STATE_NAME"]).GetString()
                                );
                        }
                    }
 
                    // add the county list to the state
                    states[i].setCounties(counties);
                }
                return states;
            }
 
            vector<Game> getGameData() {
                using namespace rapidjson;
                Document d;
 
                // request the game dataset and parse it
                d.Parse(
                    ServerComm::makeRequest("http://bridgesdata.herokuapp.com/api/games",
                {"Accept: application/json"}).c_str());
 
                const Value& D = d["data"];
                vector<Game> wrapper;
                for (SizeType i = 0; i < D.Size(); i++) {
                    const Value& V = D[i];
                    const Value& G = V["genre"];
 
                    vector<string> genre;
                    for (SizeType j = 0; j < G.Size(); j++) {
                        genre.push_back(G[j].GetString());
                    }
                    wrapper.push_back(
                        Game(   V["game"].GetString(),
                        V["platform"].GetString(),
                        V["rating"].GetDouble(),
                        genre ) );
                }
                return wrapper;
            }
            vector<ActorMovieIMDB> getActorMovieIMDBData(int number = 0) {
                using namespace rapidjson;
                Document d;
                vector<ActorMovieIMDB> wrapper;
                string url = "http://bridgesdata.herokuapp.com/api/imdb?limit=" +
                    to_string(number);
 
                // retrieve the data and parse
                d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());
 
                const Value& D = d["data"];
                for (SizeType i = 0; i < D.Size(); i++) {
                    const Value& V = D[i];
                    wrapper.push_back(
                        ActorMovieIMDB(
                            V["actor"].GetString(),
                            V["movie"].GetString()
                        )
                    );
                }
                return wrapper;
            }
 
            vector<ActorMovieIMDB> getActorMovieIMDBData2() {
                using namespace rapidjson;
                Document d;
                vector<ActorMovieIMDB> wrapper;
                string url = "http://bridgesdata.herokuapp.com/api/imdb2";
 
                // retrieve the data and parse
                d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());
 
                const Value& D = d["data"];
                for (SizeType i = 0; i < D.Size(); i++) {
                    const Value& V = D[i];
                    string actor = V["actor"].GetString();
                    string movie = V["movie"].GetString();
                    double rating = V["rating"].GetDouble();
                    const Value& G = V["genres"];
                    vector<string> genres;
                    for (SizeType j = 0; j < G.Size(); j++) {
                        genres.push_back(G[j].GetString());
                    }
                    wrapper.push_back(ActorMovieIMDB( actor, movie, (float)rating, genres));
                }
                return wrapper;
            }
 
            vector<EarthquakeUSGS> getEarthquakeUSGSData(int number = 0) {
                using namespace rapidjson;
                Document d;
                vector<EarthquakeUSGS> wrapper;
                if (number <= 0) {
                    d.Parse(ServerComm::makeRequest( "http://earthquakes-uncc.herokuapp.com/eq",
                    {"Accept: application/json"}).c_str());
                    for (SizeType i = 0; i < d.Size(); i++) {
                        const Value& V = d[i]["properties"];
                        const Value& G = d[i]["geometry"]["coordinates"];
                        wrapper.push_back(
                            EarthquakeUSGS(
                                V["mag"].GetDouble(),
                                G[0].GetDouble(),
                                G[1].GetDouble(),
                                V["place"].GetString(),
                                V["title"].GetString(),
                                V["url"].GetString(),
                                V["time"].GetString() )
                        );
                    }
                }
                else {
                    d.Parse(ServerComm::makeRequest( "http://earthquakes-uncc.herokuapp.com/eq/latest/" +
                        to_string(number), {"Accept: application/json"}).c_str());
 
                    const Value& D = d["Earthquakes"];
                    for (SizeType i = 0; i < D.Size(); i++) {
                        const Value& V = D[i]["properties"];
                        const Value& G = D[i]["geometry"]["coordinates"];
                        //              wrapper.push_back({V["mag"].GetDouble(),G[0].GetDouble(),G[1].GetDouble(),V["place"].GetString(),V["title"].GetString(),V["url"].GetString(),V["time"].GetString()});
                        wrapper.push_back(
                            EarthquakeUSGS(
                                V["mag"].GetDouble(),
                                G[0].GetDouble(),
                                G[1].GetDouble(),
                                V["place"].GetString(),
                                V["title"].GetString(),
                                V["url"].GetString(),
                                V["time"].GetString() )
                        );
                    }
                }
                return wrapper;
            }
            vector<Shakespeare> getShakespeareData(string type = "",
                bool textonly = false) {
                using namespace rapidjson;
                Document d;
                vector<Shakespeare> wrapper;
 
                string url = "http://bridgesdata.herokuapp.com/api/shakespeare/";
 
                if (type == "plays" || type == "poems")
                    url += "/" + type;
                if (textonly) {
                    url += "?format=simple";
                }
                // retrieve the data and parse
                d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());
 
                const Value& D = d["data"];
                for (SizeType i = 0; i < D.Size(); i++) {
                    const Value& V = D[i];
                    wrapper.push_back(
                        Shakespeare(
                            V["title"].GetString(),
                            V["type"].GetString(),
                            V["text"].GetString()
                        )
                    );
 
                }
                return wrapper;
            }
            Song getSong(string songTitle, string artistName = "") {
                using namespace rapidjson;
 
                Document d;
                vector<Song> wrapper;
                string url = "http://bridgesdata.herokuapp.com/api/songs/find/";
                // retrieve the data and parse
                if (songTitle.size() > 0)
                    url += songTitle;
                else {
                    throw "Incorrect use of getSong. songTitle should be given.";
                }
 
                if (artistName.size() > 0)
                    url += "?artistName=" + artistName;
 
                // check for spaces in url and replace them by '%20'
                //TODO should use curl to do that
                string::size_type n = 0;
                while ( (n = url.find(" ", n)) != string::npos) {
                    url.replace(n, 1, "%20");
                    n++;
                }
                if (debug())
                    std::cerr << "url: " << url << "\n";
                d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());
 
                string artist   = (d.HasMember("artist")) ?
                    d["artist"].GetString() : string();
                string song     = (d.HasMember("song")) ?
                    d["song"].GetString() : string();
                string album    = (d.HasMember("album"))    ?
                    d["album"].GetString() : string();
                string lyrics   = (d.HasMember("lyrics")) ?
                    d["lyrics"].GetString() : string();
                string release_date = (d.HasMember("release_date")) ?
                    d["release_date"].GetString() : string();
 
                return Song (artist, song, album, lyrics, release_date);
            }
            vector<Song> getSongData() {
                using namespace rapidjson;
 
                Document d;
                vector<Song> all_songs;
 
                string url = "http://bridgesdata.herokuapp.com/api/songs/";
                // retrieve the data and parse
 
                d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());
 
                const Value& D = d["data"];
 
                // get the songs and put them into a vector
                for (SizeType i = 0; i < D.Size(); i++) {
                    const Value& v = D[i];
 
                    //cout << v["artist"].GetString() << endl;
                    string artist   = (v.HasMember("artist")) ? v["artist"].GetString() : string();
                    string song     = (v.HasMember("song")) ? v["song"].GetString() : string();
                    string album    = (v.HasMember("album"))    ? v["album"].GetString() : string();
                    string lyrics   = (v.HasMember("lyrics")) ? v["lyrics"].GetString() : string();
                    string release_date = (v.HasMember("release_date")) ?
                        v["release_date"].GetString() : string();
                    all_songs.push_back( Song ( artist, song, album, lyrics, release_date) );
 
                }
                return all_songs;
            }
        private:
 
            GutenbergBook getAGutenbergBookMetaData(const rapidjson::Value& V) {
                using namespace rapidjson;
 
                const string s_id = V["id"].GetString();
 
                const int id = std::stoi(s_id);
 
                string title = V["title"].GetString();
 
                const Value& A = V["authors"];
                vector<string> authors;
                for (SizeType j = 0; j < A.Size(); j++)
                    authors.push_back(A[j].GetString());
 
                const Value& L = V["lang"];
                string lang = L.GetString();
 
                const Value& da = V["date_added"];
                string data_added = da.GetString();
 
                const Value& G = V["genres"];
                vector<string> genres;
                for (SizeType j = 0; j < G.Size(); j++)
                    genres.push_back(G[j].GetString());
 
                GutenbergBook gbook = GutenbergBook(title, id, authors, lang, genres, data_added);
 
                return gbook;
            }
 
        public:
            GutenbergBook getGutenbergBookMetaData(int id = 0) {
                using namespace rapidjson;
 
                // get the query string to get meta data of book
                string url = getGutenbergBaseURL() + "/meta?id=" + std::to_string(id);
 
                // make the query
                Document d;
                d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());
                // only 1 book
                return getAGutenbergBookMetaData(d["book_list"][0]);
            }
            vector<GutenbergBook> getGutenbergBookMetaData(string term, string category) {
                using namespace rapidjson;
 
                // get the query string to get meta data of book
                string url = getGutenbergBaseURL() + "/search?search=" +
                    ServerComm::encodeURLPart(term) + "&type="
                    + ServerComm::encodeURLPart(category);
                // make the query
                cout << url << endl;
                Document d;
                d.Parse(ServerComm::makeRequest(url, {"Accept: application/json"}).c_str());
 
                vector<GutenbergBook> book_list;
                int size = d["book_list"].Size();
 
                book_list.clear();
                for (int k = 0; k < size; k++)
                    book_list.push_back(getAGutenbergBookMetaData(d["book_list"][k]));
 
                return book_list;
            }
 
            string getGutenbergBookText(int id = 0)  {
                using namespace rapidjson;
 
                //URL for data request
                string data_url = getGutenbergBaseURL() + "/book?id=" + std::to_string(id);
 
                // generate the hash code - use the id
                string hash_value = "gutenberg" + std::to_string(id);
 
                // get the dataset from cache, else from the server
                string book_data = getDataSetJSON(data_url, hash_value, "gutenberg");
                // parse, and get the book text
                Document d;
                d.Parse(book_data.c_str());
 
                string s = std::to_string(id);
                return d[s.c_str()].GetString();
            }
 
            vector<CancerIncidence> getCancerIncidenceData(int num = 0) {
                using namespace rapidjson;
 
                Document d;
                vector<CancerIncidence> wrapper;
                string url = "http://bridgesdata.herokuapp.com/api/cancer/withlocations";
                if (num > 0) {
                    url += "?limit=" + to_string(num);
                }
 
                d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());
 
                // get the JSON dataset
                const Value& D = d["data"];
 
                CancerIncidence c;
                for (SizeType i = 0; i < D.Size(); i++) {
                    const Value& v = D[i];
                    const Value& age = v["Age"];
 
                    c.setAgeAdjustedRate( age["Age Adjusted Rate"].GetDouble());
                    c.setAgeAdjustedCI_Lower(age["Age Adjusted CI Lower"].GetDouble());
                    c.setAgeAdjustedCI_Upper(age["Age Adjusted CI Upper"].GetDouble());
 
                    c.setYear(v["Year"].GetInt());
 
                    const Value& data = v["Data"];
                    c.setCrudeRate(data["Crude Rate"].GetDouble());
                    c.setCrudeRate_CI_Lower(data["Crude CI Lower"].GetDouble());
                    c.setCrudeRate_CI_Upper(data["Crude CI Upper"].GetDouble());
                    c.setRace(data["Race"].GetString());
                    c.setPopulation(data["Population"].GetInt());
                    c.setEventType(data["Event Type"].GetString());
                    c.setCount(data["Count"].GetInt());
 
                    c.setAffectedArea(v["Area"].GetString());
 
                    const Value& loc = v["loc"];
                    c.setLocationX(loc[0].GetDouble());
                    c.setLocationY(loc[1].GetDouble());
 
                    wrapper.push_back(c);
                }
                return wrapper;
            }
        private:
            OSMData getOSMDataFromJSON (const string& osm_json) {
                using namespace rapidjson;
 
                Document osm_data;
 
                osm_data.Parse(osm_json.c_str());
 
                // create an osm data object
                OSMData osm;
 
                if (osm_data.HasMember("nodes")) {
                    vector<OSMVertex> vertices;
                    Value& nodes = osm_data["nodes"];
 
                    vector<long> vertex_ids;
                    // get the vertices
                    for (SizeType i = 0; i < nodes.Size(); i++) {
                        const Value& node = nodes[i];
                        OSMVertex::OSMVertexID id = node[0].GetInt64();
 
                        vertex_ids.push_back(id);
                        double lat = node[1].GetDouble(), longit = node[2].GetDouble();
                        vertices.push_back(OSMVertex(id, lat, longit));
                    }
                    osm.setVertices(vertices);
                }
                // add vertices to object
                // get the edges
 
                if (osm_data.HasMember("edges")) {
                    vector<OSMEdge> edges;
                    Value& links = osm_data["edges"];
 
                    for (SizeType i = 0; i < links.Size(); i++) {
                        const Value& link = links[i];
                        OSMVertex::OSMVertexID id1 = link[0].GetInt64();
                        OSMVertex::OSMVertexID id2 = link[1].GetInt64();
                        double dist = link[2].GetDouble();
 
                        edges.push_back(OSMEdge(id1, id2, dist));
                    }
                    osm.setEdges(edges);
                }
                // add edges to object
 
                if (osm_data.HasMember("meta")) {
                    // get lat long range
                    Value& meta = osm_data["meta"];
                    double lat_min = meta["lat_min"].GetDouble();
                    double lat_max = meta["lat_max"].GetDouble();
                    double longit_min = meta["lon_min"].GetDouble();
                    double longit_max = meta["lon_max"].GetDouble();
                    osm.setLatLongRange(lat_min, lat_max, longit_min, longit_max);
                    // get dataset name
                    osm.setName(meta["name"].GetString());
                }
                return osm;
            }
 
        public:
 
            OSMData getOSMData (double lat_min, double long_min,
                double lat_max, double long_max, string level = "default") {
 
                //URL for hash request
                string hash_url = getOSMBaseURL() +
                    "hash?minLon=" + std::to_string(long_min) +
                    "&minLat=" + std::to_string(lat_min) +
                    "&maxLon=" + std::to_string(long_max) +
                    "&maxLat=" + std::to_string(lat_max) +
                    "&level="  + ServerComm::encodeURLPart(level);
 
                //URL to request map
                string osm_url =
                    getOSMBaseURL() + "coords?minLon=" + std::to_string(long_min) +
                    "&minLat=" + std::to_string(lat_min) +
                    "&maxLon=" + std::to_string(long_max) +
                    "&maxLat=" + std::to_string(lat_max) +
                    "&level="  + ServerComm::encodeURLPart(level);
 
                // get the data set from the server or, if available, from
                // a local cache
                string osm_json = getDataSetJSON(osm_url, hash_url, "osm");
 
                //tries to get hash value for bounding box map
                return getOSMDataFromJSON(osm_json);
            }
 
            vector<Amenity>  getAmenityData(double minLat, double minLon, double
                maxLat, double maxLon, std::string amenity) {
 
                std::string amenity_url = getOSMBaseURL() + "amenity?minLon=" +
                    ServerComm::encodeURLPart(std::to_string(minLon)) +
                    "&minLat=" + ServerComm::encodeURLPart(std::to_string(minLat)) +
                    "&maxLon=" + ServerComm::encodeURLPart(std::to_string(maxLon)) +
                    "&maxLat=" + ServerComm::encodeURLPart(std::to_string(maxLat)) +
                    "&amenity=" + ServerComm::encodeURLPart(amenity);
 
                std::string hash_url = getOSMBaseURL() + "hash?minLon=" +
                    ServerComm::encodeURLPart(std::to_string(minLon)) +
                    "&minLat=" + ServerComm::encodeURLPart(std::to_string(minLat)) +
                    "&maxLon=" + ServerComm::encodeURLPart(std::to_string(maxLon)) +
                    "&maxLat=" + ServerComm::encodeURLPart(std::to_string(maxLat)) +
                    "&amenity=" + ServerComm::encodeURLPart(amenity);
 
                // make the query to the server to get a JSON of the amenities
                // implements caching to keep local copies
                string amenity_json = getDataSetJSON(amenity_url, hash_url, "amenity");
 
                // parse the data and return amenity objects
                return parseAmenityData (amenity_json);
            }
 
            vector<Amenity>  getAmenityData(const std::string& location,
                const std::string& amenity) {
                std::string amenity_url = getOSMBaseURL() + "amenity?location=" +
                    ServerComm::encodeURLPart(location) +
                    "&amenity=" + ServerComm::encodeURLPart(amenity);
 
                std::string hash_url = getOSMBaseURL() + "hash?location=" +
                    ServerComm::encodeURLPart(location) +
                    "&amenity=" + ServerComm::encodeURLPart(amenity);
 
                // make the query to the server to get a JSON of the amenities
                // implements caching to keep local copies
                string amenity_json = getDataSetJSON(amenity_url, hash_url, "amenity");
 
                // parse the data and return amenity objects
                return parseAmenityData (amenity_json);
            }
 
            vector<Amenity> parseAmenityData(string amenity_json) {
                using namespace rapidjson;
 
                vector<Amenity>  amenities;
                Document amenity_content;
 
                amenity_content.Parse(amenity_json.c_str());
                if (amenity_content.HasMember("nodes")) {
                    const Value& nodes = amenity_content["nodes"];
                    if (amenity_content.HasMember("meta")) {
                        const Value& meta = amenity_content["meta"];
 
                        // first get the meta data
                        //  amenities.setCount(meta["count"].GetInt64());
                        //  amenities.setMinLat(meta["minlat"].GetDouble());
                        //  amenities.setMinLon(meta["minlon"].GetDouble());
                        //  amenities.setMaxLat(meta["maxlat"].GetDouble());
                        //  amenities.setMaxLon(meta["maxlon"].GetDouble());
 
                        Amenity amen;
                        for (SizeType i = 0; i < nodes.Size(); i++) {
                            // get amenity data
                            const Value& node = nodes[i];
                            amen.setId(node[0].GetInt64());
                            amen.setLat(node[1].GetDouble());
                            amen.setLon(node[2].GetDouble());
                            amen.setName(node[3].GetString());
 
                            // add to the list
                            amenities.push_back(amen);
                        }
                    }
                    else {
                        cout << "meta data not found!\n";
                        throw;
                    }
                }
                else {
                    cout << "nodes data not found!\n";
                    throw;
                }
                return amenities;
            }
 
            OSMData getOSMData (string location, string level = "default") {
                //URL for hash request
                string hash_url = getOSMBaseURL() + "hash?location=" +
                    ServerComm::encodeURLPart(location) +
                    "&level=" + ServerComm::encodeURLPart(level);
 
                //URL to request map
                string osm_url = getOSMBaseURL() +
                    "loc?location=" + ServerComm::encodeURLPart(location) +
                    "&level=" + ServerComm::encodeURLPart(level);
 
                // get the data set from the server or, if available, from
                // a local cache
                string osm_json = getDataSetJSON(osm_url, hash_url, "osm");
 
                return getOSMDataFromJSON(osm_json);
            }
 
            bridges::GraphAdjList<int, std::string> getGraphFromAssignment (
                const std::string& user,
                int assignment,
                int subassignment = 0) {
 
                bridges::GraphAdjList<int, std::string> gr;
 
                std::string s = this->getAssignment(user, assignment, subassignment);
 
                rapidjson::Document doc;
                doc.Parse(s.c_str());
                if (doc.HasParseError())
                    throw "Malformed JSON";
 
                //Access doc["assignmentJSON"]
                const auto& assjson = doc.FindMember("assignmentJSON");
 
                if (assjson == doc.MemberEnd())
                    throw "Malformed GraphAdjacencyList JSON: no assignmentJSON";
 
                //Access doc["assignmentJSON"]["data"]
                const auto& dataArray = assjson->value.FindMember("data");
 
                if (dataArray == assjson->value.MemberEnd()
                    || dataArray->value.IsArray() == false)
                    throw "Malformed GraphAdjacencyList JSON: No data";
 
                const auto& data = dataArray->value.GetArray()[0];
 
                //Access doc["assignmentJSON"]["data"][0]["visual"]
                const auto& dataVisual = data.FindMember("visual");
 
                if (dataVisual == data.MemberEnd() ||
                    dataVisual->value.IsString() == false)
                    throw "Malformed GraphAdjacencyList JSON";
 
                std::string assignment_type = dataVisual->value.GetString();
 
                if (assignment_type != "GraphAdjacencyList")
                    throw "Malformed GraphAdjacencyList JSON: Not a GraphAdjacencyList";
 
                //reconstructing vertices out of nodes, and using the optional "name" as the data associated
                {
                    const auto& nodes = data.FindMember("nodes");
                    if (nodes == data.MemberEnd() ||
                        nodes->value.IsArray() == false)
                        throw "Malformed GraphAdjacencyList JSON: malformed nodes";
 
                    const auto& nodeArray = nodes->value.GetArray();
                    int nbVertex = nodeArray.Size();
                    for (int i = 0; i < nbVertex; ++i) {
                        std::string name;
 
                        const auto& vertexJSONstr = nodeArray[i];
 
                        const auto& nameJSON = vertexJSONstr.FindMember("name");
                        if (nameJSON != vertexJSONstr.MemberEnd()
                            && nameJSON->value.IsString()) {
                            name = nameJSON->value.GetString();
                        }
                        gr.addVertex(i, name);
                    }
                }
 
                //reconstructing links, and using "label" as data associated with the link
                {
                    const auto& links = data.FindMember("links");
                    if (links == data.MemberEnd() ||
                        links->value.IsArray() == false)
                        throw "Malformed GraphAdjacencyList JSON: malformed links";
 
                    const auto& linkArray = links->value.GetArray();
                    int nbLink = linkArray.Size();
                    for (int i = 0; i < nbLink; ++i) {
                        std::string name;
                        int src;
                        int dest;
                        int wgt;
 
                        const auto& linkJSONstr = linkArray[i];
 
                        //checking label. Maybe does not exist? (is that field optional?)
                        const auto& nameJSON = linkJSONstr.FindMember("label");
                        if (nameJSON != linkJSONstr.MemberEnd()
                            && nameJSON->value.IsString()) {
                            name = nameJSON->value.GetString();
                        }
 
                        //checking source
                        const auto& srcJSON = linkJSONstr.FindMember("source");
                        if (srcJSON == linkJSONstr.MemberEnd()
                            || srcJSON->value.IsInt() == false) {
                            throw "Malformed GraphAdjacencyList JSON: malformed link";
                        }
                        src = srcJSON->value.GetInt();
 
                        //checking destination
                        const auto& dstJSON = linkJSONstr.FindMember("target");
                        if (dstJSON == linkJSONstr.MemberEnd()
                            || dstJSON->value.IsInt() == false) {
                            throw "Malformed GraphAdjacencyList JSON: malformed link";
                        }
                        dest = dstJSON->value.GetInt();
 
                        //checking weight. //why is weight a mandatory parameter?
                        const auto& wgtJSON = linkJSONstr.FindMember("weight");
                        if (wgtJSON == linkJSONstr.MemberEnd()
                            || wgtJSON->value.IsInt() == false) {
                            throw "Malformed GraphAdjacencyList JSON: malformed link";
                        }
                        wgt = wgtJSON->value.GetInt();
 
                        //adding edge.
                        gr.addEdge(src, dest, name);
                    }
                }
 
                return gr;
            }
 
            bridges::ColorGrid getColorGridFromAssignment(const std::string& user,
                int assignment,
                int subassignment = 0) {
 
                std::string s = this->getAssignment(user, assignment, subassignment);
 
                rapidjson::Document doc;
                doc.Parse(s.c_str());
                if (doc.HasParseError())
                    throw "Malformed JSON";
 
                try {
                    std::string assignment_type = doc["assignment_type"].GetString();
 
                    if (assignment_type != "ColorGrid")
                        throw "Malformed ColorGrid JSON: Not a ColorGrid";
                }
                catch (rapidjson_exception re) {
                    throw "Malformed JSON: Not a Bridges assignment?";
                }
 
                try {
                    auto& data = doc["data"][0];
 
                    std::string encoding = data["encoding"].GetString();
                    if (encoding != "RAW" && encoding != "RLE")
                        throw "Malformed ColorGrid JSON: encoding not supported";
 
                    //Access doc["data"][0]["dimensions"]
                    const auto& dimensions = data["dimensions"];
                    int dimx = dimensions[0].GetInt();
                    int dimy = dimensions[1].GetInt();
 
                    if (debug())
                        std::cerr << "Dimensions: " << dimx << "x" << dimy << std::endl;
 
                    //Access doc["data"][0]["nodes"][0]
                    std::string base64_encoded_assignment = data["nodes"][0].GetString();
 
                    std::vector<bridges::BYTE> decoded = bridges::base64::decode(base64_encoded_assignment);
 
                    bridges::ColorGrid cg (dimx, dimy);
 
                    if (encoding == "RAW") {
                        if (debug())
                            std::cerr << "decoding RAW" << std::endl;
                        if (debug())
                            std::cerr << "length: " << decoded.size() << std::endl;
                        if (decoded.size() < dimx * dimy * 4)
                            throw "Malformed ColorGrid JSON: nodes is smaller than expected";
 
                        //first pixel
                        //std::cerr<<(int)decoded[0]<<" "<<(int)decoded[1]<<" "<<(int)decoded[2]<<" "<<(int)decoded[3]<<std::endl;
 
                        //bridges::ColorGrid* ptr = new bridges::ColorGrid (dimx, dimy);
 
                        size_t base = 0;
 
                        for (int x = 0; x < dimx; ++x) {
                            for (int y = 0; y < dimy; ++y) {
                                bridges::Color c ((int)decoded[base],
                                    (int)decoded[base + 1],
                                    (int)decoded[base + 2],
                                    (int)decoded[base + 3]
                                );
 
                                cg.set(x, y, c);
                                base += 4;
                            }
                        }
                    }
                    else if (encoding == "RLE") {
                        if (debug())
                            std::cerr << "Decoding RLE" << std::endl;
 
                        int currentInDecoded = 0;
                        int currentInCG = 0;
                        while (currentInDecoded != decoded.size()) {
                            if (currentInDecoded + 5 > decoded.size())
                                throw "Malformed ColorGrid JSON: nodes is not a multiple of 5";
 
                            int repeat = (BYTE) decoded[currentInDecoded++];
                            int r = (BYTE) decoded[currentInDecoded++];
                            int g = (BYTE) decoded[currentInDecoded++];
                            int b = (BYTE) decoded[currentInDecoded++];
                            int a = (BYTE) decoded[currentInDecoded++];
 
                            if (debug())
                                std::cerr << "indecoded: " << currentInDecoded
                                    << " repeat: " << (int)repeat
                                    << " color(" << (int)r << "," << (int)g << "," << (int)b << "," << (int)a << ")"
                                    << std::endl;
 
                            bridges::Color c (r, g, b, a);
 
                            while (repeat >= 0) {
                                int posX = currentInCG / dimy;
                                int posY = currentInCG % dimy;
                                if (posX >= dimx || posY >= dimy) {
                                    if (debug())
                                        std::cerr << posX << " " << dimx << " " << posY << " " << dimy << std::endl;
                                    throw "Malformed ColorGrid JSON: Too much data in nodes";
                                }
                                cg.set(posX, posY, c);
 
                                currentInCG++;
                                repeat --;
                            }
                        }
                        if (debug())
                            std::cerr << "written " << currentInCG << " pixels" << std::endl;
                        if (currentInCG != dimx * dimy)
                            throw "Malformed ColorGrid JSON: Not enough data in nodes";
                    }
 
                    return cg;
                }
                catch (rapidjson_exception re) {
                    throw "Malformed ColorGrid JSON";
                }
 
            }
        private:
            /***
             * This function obtains the JSON representation of a particular subassignment.
             *
             * @return a string that is the JSON representation of the subassignment as stored by the Bridges server.
             * @param user the name of the user who uploaded the assignment
             * @param assignment the ID of the assignment to get
             * @param subassignment the ID of the subassignment to get
             ***/
            std::string getAssignment(std::string user,
                int assignment,
                int subassignment = 0) {
                std::vector<std::string> headers;
 
                std::stringstream ss;
 
                if (bridges_inst)
                    ss << bridges_inst->getServerURL();
                else
                    ss << bridges::Bridges::getDefaultServerURL();
                ss << "/assignmentJSON/"
                    << assignment << ".";
                ss << std::setfill('0') << std::setw(2) << subassignment;
                ss << "/" << user;
 
                std::string url = ss.str();
 
                std::string s = bridges::ServerComm::makeRequest(url, headers);
 
                return s;
            }
 
            void removeFirstOccurence (std::string & str, const std::string & toRemove) {
                size_t pos = str.find(toRemove);
                if (pos  != std::string::npos) {
                    str.erase(pos, toRemove.length());
                }
            }
 
            void  getWikidataActorMovieDirect (int yearbegin, int yearend, std::vector<MovieActorWikidata>& vout) {
                std::string codename = "wikidata-actormovie-" + std::to_string(yearbegin) + "-" + std::to_string(yearend);
                std::string json;
                bool from_cache = false;
                try {
                    if (my_cache.inCache(codename)) {
                        json = my_cache.getDoc(codename);
                        from_cache = true;
                    }
                }
                catch (CacheException& ce) {
                    //something went bad trying to access the cache
                    std::cout << "Exception while reading from cache. Ignoring cache and continue.\n( What was:" << ce.what() << ")" << std::endl;
                }
 
                if (!from_cache) {
                    std::vector<std::string> http_headers;
                    http_headers.push_back("User-Agent: bridges-cxx"); //wikidata kicks you out if you don't have a useragent
                    http_headers.push_back("Accept: application/json"); //tell wikidata we are OK with JSON
 
                    string url = "https://query.wikidata.org/sparql?";
 
                    //Q1860 is "English"
                    //P364 is "original language of film or TV show"
                    //P161 is "cast member"
                    //P577 is "publication date"
                    //A11424 is "film"
                    //P31 is "instance of"
                    // "instance of film" is necessary to filter out tv shows
                    std::string sparqlquery =
                        "SELECT ?movie ?movieLabel ?actor ?actorLabel WHERE \
{\
  ?movie wdt:P31 wd:Q11424.\
  ?movie wdt:P161 ?actor.\
  ?movie wdt:P364 wd:Q1860.\
  ?movie wdt:P577 ?date.\
  FILTER(YEAR(?date) >= " + std::to_string(yearbegin) + " && YEAR(?date) <= " + std::to_string(yearend) + ").\
    SERVICE wikibase:label { bd:serviceParam wikibase:language \"en\". } \
}";
                    url += "query=" + ServerComm::encodeURLPart(sparqlquery);
                    url += "&";
                    url += "format=json";
 
                    if (debug()) {
                        std::cout << "URL: " << url << "\n";
                    }
                    // get the Wikidata json
                    json = ServerComm::makeRequest(url, http_headers);
 
                    try {
                        my_cache.putDoc(codename, json);
                    }
                    catch (CacheException& ce) {
                        //something went bad trying to access the cache
                        std::cerr << "Exception while storing in cache. Weird but not critical. (What was: " << ce.what() << " )" << std::endl;
                    }
                }
 
                {
                    using namespace rapidjson;
                    rapidjson::Document doc;
                    doc.Parse(json.c_str());
                    if (doc.HasParseError())
                        throw "Malformed JSON";
 
                    try {
                        const auto& resultsArray = doc["results"]["bindings"].GetArray();
 
                        for (auto& mak_json : resultsArray) {
                            MovieActorWikidata mak;
 
                            // all wikidata uri start with "http://www.wikidata.org/entity/"
                            // so strip it out because it does not help discriminate and
                            // consume memory and runtime to compare string
                            std::string actoruri = mak_json["actor"]["value"].GetString();
                            std::string movieuri = mak_json["movie"]["value"].GetString();
                            removeFirstOccurence (actoruri, "http://www.wikidata.org/entity/");
 
                            removeFirstOccurence (movieuri, "http://www.wikidata.org/entity/");
 
                            mak.setActorURI(actoruri);
                            mak.setMovieURI(movieuri);
                            mak.setActorName(mak_json["actorLabel"]["value"].GetString());
                            mak.setMovieName(mak_json["movieLabel"]["value"].GetString());
                            vout.push_back(mak);
                        }
 
                    }
                    catch (rapidjson_exception re) {
                        throw "Malformed JSON: Not from wikidata?";
                    }
                }
            }
        public:
 
            std::vector<MovieActorWikidata> getWikidataActorMovie (int yearbegin, int yearend) {
                //Internally this function get the data year by year. This
                //is pretty bad because it hits wikidata the first time
                //for multiple years. But it enables to work around
                //wikidata's time limit.  This also works well because the
                //Cache will store each year independently and so without
                //redundancy.  Though I (Erik) am not completely sure that a
                //movie can be appear in different years, for instance it
                //can be released in the US in 2005 but in canada in
                //2006...
 
                std::vector<MovieActorWikidata> ret;
                for (int y = yearbegin; y <= yearend; ++y) {
                    // cout << "getting year " << y << endl;
                    getWikidataActorMovieDirect (y, y, ret);
                }
                return ret;
            }
 
            ElevationData getElevationData (
                double minLat, double minLon,
                double maxLat, double maxLon, double res = 0.0166)  {
 
                // set up the elevation data url to get the data, given
                // a lat/long bounding box
 
                std::string elev_url = getElevationBaseURL() +
                    "elevation?minLon=" + ServerComm::encodeURLPart(std::to_string(minLon)) +
                    "&minLat=" + ServerComm::encodeURLPart(std::to_string(minLat)) +
                    "&maxLon=" + ServerComm::encodeURLPart(std::to_string(maxLon)) +
                    "&maxLat=" + ServerComm::encodeURLPart(std::to_string(maxLat)) +
                    "&resX=" + ServerComm::encodeURLPart(std::to_string(res)) +
                    "&resY=" + ServerComm::encodeURLPart(std::to_string(res));
 
                if (debug())
                    cout << "Elevation URL:" << elev_url << "\n";
                cout << "Elevation URL:" << elev_url << "\n";
 
                std::string hash_url = getElevationBaseURL() +
                    "hash?minLon=" + ServerComm::encodeURLPart(std::to_string(minLon)) +
                    "&minLat=" + ServerComm::encodeURLPart(std::to_string(minLat)) +
                    "&maxLon=" + ServerComm::encodeURLPart(std::to_string(maxLon)) +
                    "&maxLat=" + ServerComm::encodeURLPart(std::to_string(maxLat)) +
                    "&resX=" + ServerComm::encodeURLPart(std::to_string(res)) +
                    "&resY=" + ServerComm::encodeURLPart(std::to_string(res));
 
                if (debug())
                    cout << "Hash URL:" << hash_url << "\n";
 
                // get the dataset's JSON from the local cache, if available,
                // else from the server
 
                string elev_json = getDataSetJSON(elev_url, hash_url, "elevation"); //Erik says: we call that function but the format ain't JSON somehow.
 
                return parseElevationData(elev_json);
            }
 
            ElevationData parseElevationData (string elev_json) {
 
                // use a string stream to parse the data, which is not really a JSON,
                // but raw text
                stringstream ss(elev_json);
 
                int rows, cols, elev_val;
                double ll_x, ll_y, cell_size;
                string tmp;
 
                // get the dimensions, origin
                ss >> tmp >> cols >> tmp >> rows >>
                    tmp >> ll_x >> tmp >> ll_y >>
                    tmp >> cell_size;
 
                if (!ss)
                    throw "Parse Error";
 
 
                // create the elevation object
                ElevationData elev_data (rows, cols);
                elev_data.setxll(ll_x);
                elev_data.setyll(ll_y);
                elev_data.setCellSize(cell_size);
 
                // load the elevation data
                for (int i = 0; i < rows; i++) {
                    for (int j = 0; j < cols; j++) {
                        ss >> elev_val;
                        elev_data.setVal(i, j, elev_val);
                    }
                }
                if (!ss)
                    throw "Parse Error";
 
                return elev_data;
            }
 
            std::vector<std::string> getAvailableSubreddits() {
                string base_url = getRedditURL();
                string url = base_url + "/listJSON";
                if (debug()) {
                    std::cout << "hitting url: " << url << "\n";
                }
                using namespace rapidjson;
                Document doc;
                {
                    std::string s = ServerComm::makeRequest(url, {"Accept: application/json"});
                    if (debug()) {
                        std::cout << "Returned JSON:" << s << "\n";
                    }
                    try {
                        doc.Parse(s.c_str());
                    }
                    catch (rapidjson_exception& re) {
                        std::cerr << "malformed subreddit list" << "\n";
                        std::cerr << "Original exception: " << (std::string)re << "\n";
                    }
                }
 
                std::vector<std::string> subreddits;
                try {
                    for (auto& m : doc.GetArray()) {
 
                        std::string subred = m.GetString();
                        subreddits.push_back(subred);
 
                    }
                }
                catch (rapidjson_exception& re) {
                    std::cerr << "malformed subreddit list" << "\n";
                    std::cerr << "Original exception: " << (std::string)re << "\n";
                }
 
                return subreddits;
 
            }
 
            vector<Reddit> getRedditData(string subreddit, int time_request = -9999) {
                string base_url = getRedditURL();
                if (debug()) {
                    cout <<  "reddit base url:" << base_url <<  "\n";
                }
                string url = base_url + "/cache?subreddit=" + subreddit +
                    "&time_request=" + std::to_string(time_request);
 
                if (debug()) {
                    cout <<  "reddit url:" << url <<  "\n";
                }
 
                using namespace rapidjson;
                Document doc;
                {
                    std::string s = ServerComm::makeRequest(url, {"Accept: application/json"});
                    if (debug()) {
                        std::cout << "Returned JSON:" << s << "\n";
                    }
                    doc.Parse(s.c_str());
                }
 
                vector<Reddit> reddit_posts;
                for (auto& m : doc.GetObject()) {
                    try {
                        if (debug()) {
                            std::cout << m.name.GetString() << "\n";
                        }
                        auto& postJSON = m.value;
 
                        std::string id = postJSON["id"].GetString();
                        std::string title = postJSON["title"].GetString();
                        std::string author = postJSON["author"].GetString();
                        int score = postJSON["score"].GetInt();
                        float vote_ratio = postJSON["vote_ratio"].GetDouble();
                        int comment_count = postJSON["comment_count"].GetInt();
                        std::string subreddit = postJSON["subreddit"].GetString();
                        int posttime = postJSON["post_time"].GetDouble();
                        std::string url = postJSON["url"].GetString();
                        std::string text = postJSON["text"].GetString();
 
                        Reddit r;
                        r.setID(id);
                        r.setTitle(title);
                        r.setAuthor(author);
                        r.setScore(score);
                        r.setVoteRatio(vote_ratio);
                        r.setCommentCount(comment_count);
                        r.setSubreddit(subreddit);
                        r.setPostTime(posttime);
                        r.setURL(url);
                        r.setText(text);
                        reddit_posts.push_back(r);
                    }
                    catch (rapidjson_exception& re) {
                        std::cerr << "malformed Reddit post" << "\n";
                        std::cerr << "Original exception: " << (std::string)re << "\n";
                    }
                }
 
                return reddit_posts;
            }
 
        private:
            string getHashCode (string hash_url, string data_type) {
                string hash_value;
                if (data_type == "osm" || data_type == "amenity" ||
                    data_type == "elevation") {
                    hash_value = ServerComm::makeRequest(hash_url, {"Accept: application/json"});
                }
                else if (data_type == "gutenberg")
                    hash_value = hash_url;
 
                return hash_value;
            }
 
            std::string getDataSetJSON(std::string data_url, std::string hash_url,
                std::string data_type) {
 
                std::string data_json = "";
 
                // First check to see if the requested data is stored in local cache
                // get hash value for elevation data
                if (debug())
                    cerr << "Checking the cache: Hash url: " << hash_url << "\n";
 
                // generate the hash code
                string hash_value = getHashCode(hash_url, data_type);
 
                bool dataloaded = false;
 
                if ((hash_value != "false") && (my_cache.inCache(hash_value) == true)) { //local cache contains the dataset
                    try {
                        data_json = my_cache.getDoc(hash_value);
                        dataloaded = true;
                    }
                    catch (CacheException& ce) {
                        //something went bad trying to access the data in the local cache
                        std::cout << "Exception while reading from cache. "
                            << "Ignoring cache and continuing..\n (What was:" << ce.what() << ")\n";
                    }
                }
                if (!dataloaded) {
                    //Data could not get accessed from cache for some reason.
                    //So teh data need to be access from the remote server.
                    //Then we will store it in a local cache for future usage.
                    if (debug())
                        std::cerr << "Hitting data URL: " << data_url << "\n";
 
                    //Requests the data
                    data_json = ServerComm::makeRequest(data_url,
                    {"Accept: application/json"});
 
                    //Store the data in cache for future reuse
                    try {
                        // We need the data's hash code to know where to store it in local cache.
                        // We may already have it from the previous query.
                        if (hash_value == "false") {
                            if (debug())
                                std::cerr << "Hitting hash URL: " << hash_value << "\n";
 
                            hash_value = getHashCode(hash_url, data_type);
                        }
 
                        // This test should only ever be true if something wrong happens server-side
                        if (hash_value == "false") {
                            std::cerr << "Error while gathering hash value for " << data_type << " dataset..\n"
                                << "Weird but not critical.\n";
                        }
                        else {
                            my_cache.putDoc(hash_value, data_json);
                        }
                    }
                    catch (CacheException& ce) {
                        //something went bad trying to access the cache
                        std::cerr << "Exception while storing in cache. " <<
                                     "Weird but not critical.\n" <<
                                     "(What was: " << ce.what() << ")\n";
                        if (debug())
                            std::cerr << "Tried to store hash=" << hash_value <<
                                         " key = " << data_json << std::endl;
                    }
                }
 
                if (debug())
                    cout << data_json;
 
                return data_json;
            }
 
    }; // class DataSource
} // namespace bridges
#endif