Bridges-C++ 3.5.0-dev2-1-ge3e57bf
Bridges(C++ API)
DataSource.h
Go to the documentation of this file.
1#ifndef DATA_SOURCE_H
2#define DATA_SOURCE_H
3
4#include <vector>
5#include <string>
6#include <unordered_map>
7#include <set>
8
9using namespace std;
10
11#include <JSONutil.h>
13#include "./data_src/Game.h"
18#include "./data_src/Song.h"
20#include "./data_src/OSMData.h"
22#include "./data_src/OSMEdge.h"
24#include "./data_src/Amenity.h"
25#include "./data_src/Reddit.h"
26#include "./data_src/City.h"
27#include "./data_src/USState.h"
28#include "./data_src/USCounty.h"
29#include "./data_src/Country.h"
30#include "MapConstants.h"
31#include "ColorGrid.h"
32#include "base64.h"
33#include <GraphAdjList.h>
34#include <ServerComm.h>
35#include <Bridges.h>
36#include "rapidjson/document.h"
37#include <rapidjson/istreamwrapper.h>
38#include "assert.h"
39#include "rapidjson/error/en.h"
40
41#include <fstream>
42#include <sys/types.h>
43#include <sys/stat.h>
44#include <stdio.h>
45
46#include <Cache.h>
47
48
49namespace bridges {
50 using namespace bridges::dataset;
51 using namespace bridges::datastructure;
52
69 class DataSource {
70
71 private:
72
73 bool debug_flag = false;
74
75 int debug() const {
76 return debug_flag;
77 }
78
79 void set_debug_flag() {
80 debug_flag = true;
81 }
82
83 bridges::Bridges* bridges_inst;
84 bridges::lruCache my_cache;
85
86 string getOSMBaseURL() const {
87 if (sourceType == "local")
88 return "http://localhost:3000/";
89
90 return "http://osm-data.bridgesuncc.org/";
91 }
92 string getElevationBaseURL() const {
93 if (sourceType == "local")
94 return "http://localhost:3000/";
95
96 return "http://elevation-data.bridgesuncc.org/";
97 }
98 string getGutenbergBaseURL() const {
99 if (sourceType == "local")
100 return "http://localhost:3000/";
101
102 if (sourceType == "testing")
103 return "http://gutenberg-t-data.bridgesuncc.org/";
104
105 return "http://gutenberg-data.bridgesuncc.org/";
106 }
107 string getRedditURL() {
108 if (sourceType == "testing")
109 return "http://reddit-t-data.bridgesuncc.org";
110 else if (sourceType == "local")
111 return "http://localhost:9999";
112 else
113 return "http://reddit-data.bridgesuncc.org";
114 }
115
116 string sourceType = "live";
117
118 string getUSCitiesURL() {
119 return "http://static-data.bridgesuncc.org/api/us_cities";
120 }
121
122 string getWorldCitiesURL() {
123 return "http://static-data.bridgesuncc.org/api/world_cities";
124 }
125
126 string getUSStateCountiesURL() {
127 return "http://static-data.bridgesuncc.org/api/us_map?state=";
128 }
129
130 string getWorldCountriesURL() {
131 return "http://static-data.bridgesuncc.org/api/world_map";
132 }
133
134 void defaultDebug() {
135 char* force = getenv("FORCE_BRIDGES_DATADEBUG");
136 if (force != nullptr)
137 set_debug_flag();
138
139 }
140
141 public:
143 : bridges_inst(br), my_cache(120) {
144 defaultDebug();
145 }
146
148 : DataSource(&br) {
149 defaultDebug();
150 }
151
159 void setSourceType(string type) {
160 if ( !((type == "live") || (type == "testing") || (type == "local")))
161 throw "Incorrect data server type. Must be live, testing or local";
162 if ((type == "testing") || (type == "local"))
163 debug_flag = true;
164
165 sourceType = type;
166 }
167
190 vector<City> getUSCities (unordered_map<string, string> params) {
191 // check the parameters for illegal keys
192 // get the legal keys into a set
193 set<string> keys;
194 keys.insert("city");
195 keys.insert("state");
196 keys.insert("country");
197 keys.insert("min_elev");
198 keys.insert("max_elev");
199 keys.insert("min_pop");
200 keys.insert("max_pop");
201 keys.insert("min_long");
202 keys.insert("max_long");
203 keys.insert("min_lat");
204 keys.insert("max_lat");
205 keys.insert("limit");
206 keys.insert("time_zone");
207
208 unordered_map<string, string>::iterator it;
209 for (it = params.begin(); it != params.end(); it++) {
210 if (keys.find(it->first) == keys.end())
211 throw std::invalid_argument ("\n\nKey value : " + it->first +
212 " incorrect\n\n Legal key values: \n 'city', 'state', 'country', 'min_lat', 'max_lat', 'min_long', 'max_long', 'min_pop', 'max_pop', 'time_zone', 'limit' ");
213 }
214
215 string url = getUSCitiesURL() + "?";
216 if (params.find("city") != params.end())
217 url += "city=" + ServerComm::encodeURLPart(params["city"]) + "&";
218 if (params.find("state") != params.end())
219 url += "state=" + ServerComm::encodeURLPart(params["state"]) + "&";
220 if (params.find("country") != params.end())
221 url += "country=" + ServerComm::encodeURLPart(params["country"]) + "&";
222 if (params.find("min_lat") != params.end())
223 url += "minLat=" + ServerComm::encodeURLPart(params["min_lat"]) + "&";
224 if (params.find("max_lat") != params.end())
225 url += "maxLat=" + ServerComm::encodeURLPart(params["max_lat"]) + "&";
226 if (params.find("min_long") != params.end())
227 url += "minLong=" + ServerComm::encodeURLPart(params["min_long"]) + "&";
228 if (params.find("max_long") != params.end())
229 url += "maxLong=" + ServerComm::encodeURLPart(params["max_long"]) + "&";
230 if (params.find("min_elev") != params.end())
231 url += "minElevation=" + ServerComm::encodeURLPart(params["min_elev"]) + "&";
232 if (params.find("max_elev") != params.end())
233 url += "maxElevation=" + ServerComm::encodeURLPart(params["max_elev"]) + "&";
234 if (params.find("min_pop") != params.end())
235 url += "minPopulation=" + ServerComm::encodeURLPart(params["min_pop"]) + "&";
236 if (params.find("maxPopulation") != params.end())
237 url += "max_pop=" + ServerComm::encodeURLPart(params["max_pop"]) + "&";
238 if (params.find("limit") != params.end())
239 url += "limit=" + ServerComm::encodeURLPart(params["limit"]) + "&";
240
241 // remove the last &
242 url = url.substr(0, url.length() - 1);
243
244 // make the request
245 using namespace rapidjson;
246 Document doc;
247 doc.Parse(
248 ServerComm::makeRequest(url, {"Accept: application/json"}).c_str()
249 );
250
251 // parse the json
252 const Value& city_json = doc["data"];
253 vector<City> us_cities;
254 for (SizeType i = 0; i < city_json.Size(); i++) {
255 const Value& val = city_json[i];
256 us_cities.push_back (
257 City(
258 val["city"].GetString(),
259 val["state"].GetString(),
260 val["country"].GetString(),
261 val["timezone"].GetString(),
262 val["elevation"].GetInt(),
263 val["population"].GetInt(),
264 val["lat"].GetDouble(),
265 val["lon"].GetDouble()
266 ));
267 }
268
269 return us_cities;
270 }
271
277 vector<USState> getUSMapData () {
278 return getUSMapCountyData(all_us_states, false);
279 }
280
287 vector<USState> getUSMapCountyData () {
288 return getUSMapCountyData(all_us_states, true);
289 }
290
299 vector<USState> getUSMapCountyData (vector<string> state_names,
300 bool view_counties = true) {
301
302 // form the http query
303 string url = getUSStateCountiesURL();
304 for (auto& k : state_names)
305 url += ServerComm::encodeURLPart(k) + ',';
306
307 // remove the last comma
308 url = url.substr(0, url.size() - 1);
309
310 if (debug())
311 std::cerr << "Hitting: " << url << std::endl;
312
313 // make the request
314 using namespace rapidjson;
315 Document doc;
316 doc.Parse(
317 ServerComm::makeRequest(url, {"Accept: application/json"}).c_str()
318 );
319
320 // iterate through the states to parse and store in
321 // the State and County objects
322 vector<USState> states;
323 const Value& state_data = doc["data"];
324 for (SizeType i = 0; i < state_names.size(); i++) {
325 const Value& st = state_data[i];
326 const Value& county_data = st["counties"];
327 const Value& st_name = st["_id"]["input"];
328
329 // create the state object and add it to a list
330 states.push_back(USState(st_name.GetString()));
331 unordered_map<string, USCounty> counties = states[i].getCounties();
332
333 states[i].setViewCountiesFlag(view_counties);
334
335 // get county data
336 if (view_counties) {
337 for (SizeType j = 0; j < county_data.Size(); j++) {
338 const Value& val = county_data[j];
339 // get its geoid
340 string geoid = (val["properties"]["GEOID"]).GetString();
341 counties[geoid] = USCounty(geoid,
342 (val["properties"]["FIPS_CODE"]).GetString(),
343 (val["properties"]["COUNTY_STATE_CODE"]).GetString(),
344 (val["properties"]["COUNTY_STATE_NAME"]).GetString()
345 );
346 }
347 }
348
349 // add the county list to the state
350 states[i].setCounties(counties);
351 }
352 return states;
353 }
354 /*
355 * See tutorial at https://bridgesuncc.github.io/tutorials/??
356 *
357 * @brief Gets the countrydata for all world countries
358 *
359 * Currently reads from a data file (json), which will be pushed to a URL
360 * end point
361 * at a later point
362 *
363 * @returns vector of country data in Country objects
364 */
365 vector<Country> getWorldMapData() {
367 }
368 vector<Country> getWorldMapData(vector<string> countries) {
369
370 vector<Country> country_data;
371
372 if (countries[0] == "all") // all countries included
373 countries = all_countries;
374
375 string url = getWorldCountriesURL();
376
377 if (debug())
378 std::cerr << "Hitting: " << url << std::endl;
379
380 // make the request
381 using namespace rapidjson;
382 Document doc;
383 doc.Parse(
384 ServerComm::makeRequest(url, {"Accept: application/json"}).c_str()
385 );
386
387 // parse the JSON, put the countries by name into a map
388 // makes it easier to extract a subset of countries
389 unordered_map<string, Country> country_map;
390 const Value& country_json = doc["data"];
391 for (SizeType i = 0; i < country_json.Size(); i++) {
392 const Value& cval = country_json[i];
393 string name = string(cval["name"].GetString());
394 country_map[name] = Country(
395 string(cval["name"].GetString()),
396 string(cval["alpha-2"].GetString()),
397 string(cval["alpha-3"].GetString()),
398 cval["numeric-3"].GetInt(),
399 datastructure::Color("lightblue"),
400 datastructure::Color("green"),
401 1.
402 );
403 }
404 // put the country info into a vector
405 for (auto c : countries) {
406 country_data.push_back(country_map[c]);
407 }
408 return country_data;
409 }
410
423 vector<Game> getGameData() {
424 using namespace rapidjson;
425 Document d;
426
427 // request the game dataset and parse it
428 d.Parse(
429 ServerComm::makeRequest("http://static-data.bridgesuncc.org/api/games",
430 {"Accept: application/json"}).c_str());
431
432 const Value& D = d["data"];
433 vector<Game> wrapper;
434 for (SizeType i = 0; i < D.Size(); i++) {
435 const Value& V = D[i];
436 const Value& G = V["genre"];
437
438 vector<string> genre;
439 for (SizeType j = 0; j < G.Size(); j++) {
440 genre.push_back(G[j].GetString());
441 }
442 wrapper.push_back(
443 Game( V["game"].GetString(),
444 V["platform"].GetString(),
445 V["rating"].GetDouble(),
446 genre ) );
447 }
448 return wrapper;
449 }
461 vector<ActorMovieIMDB> getActorMovieIMDBData(int number = 0) {
462 using namespace rapidjson;
463 Document d;
464 vector<ActorMovieIMDB> wrapper;
465 string url = "http://static-data.bridgesuncc.org/api/imdb?limit=" +
466 to_string(number);
467
468 // retrieve the data and parse
469 d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());
470
471 const Value& D = d["data"];
472 for (SizeType i = 0; i < D.Size(); i++) {
473 const Value& V = D[i];
474 wrapper.push_back(
476 V["actor"].GetString(),
477 V["movie"].GetString()
478 )
479 );
480 }
481 return wrapper;
482 }
483
494 vector<ActorMovieIMDB> getActorMovieIMDBData2() {
495 using namespace rapidjson;
496 Document d;
497 vector<ActorMovieIMDB> wrapper;
498 string url = "http://static-data.bridgesuncc.org/api/imdb2";
499
500 // retrieve the data and parse
501 d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());
502
503 const Value& D = d["data"];
504 for (SizeType i = 0; i < D.Size(); i++) {
505 const Value& V = D[i];
506 string actor = V["actor"].GetString();
507 string movie = V["movie"].GetString();
508 double rating = V["rating"].GetDouble();
509 const Value& G = V["genres"];
510 vector<string> genres;
511 for (SizeType j = 0; j < G.Size(); j++) {
512 genres.push_back(G[j].GetString());
513 }
514 wrapper.push_back(ActorMovieIMDB( actor, movie, (float)rating, genres));
515 }
516 return wrapper;
517 }
518
532 vector<EarthquakeUSGS> getEarthquakeUSGSData(int number = 0) {
533 using namespace rapidjson;
534 Document d;
535 vector<EarthquakeUSGS> wrapper;
536 if (number <= 0) {
537 d.Parse(ServerComm::makeRequest( "http://earthquakes-data.bridgesuncc.org/eq",
538 {"Accept: application/json"}).c_str());
539 }
540 else {
541 d.Parse(ServerComm::makeRequest( "http://earthquakes-data.bridgesuncc.org/eq/latest/" +
542 to_string(number), {"Accept: application/json"}).c_str());
543
544 }
545 const Value& D = d["Earthquakes"];
546 for (SizeType i = 0; i < D.Size(); i++) {
547 const Value& V = D[i]["properties"];
548 const Value& G = D[i]["geometry"]["coordinates"];
549 wrapper.push_back(
551 V["mag"].GetDouble(),
552 G[0].GetDouble(),
553 G[1].GetDouble(),
554 V["place"].GetString(),
555 V["title"].GetString(),
556 V["url"].GetString(),
557 V["time"].GetString() )
558 );
559 }
560
561 return wrapper;
562 }
578 vector<Shakespeare> getShakespeareData(string type = "",
579 bool textonly = false) {
580 using namespace rapidjson;
581 Document d;
582 vector<Shakespeare> wrapper;
583
584 string url = "http://static-data.bridgesuncc.org/api/shakespeare/";
585
586 if (type == "plays" || type == "poems")
587 url += "/" + ServerComm::encodeURLPart(type);
588 if (textonly) {
589 url += "?format=simple";
590 }
591 // retrieve the data and parse
592 d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());
593
594 const Value& D = d["data"];
595 for (SizeType i = 0; i < D.Size(); i++) {
596 const Value& V = D[i];
597 wrapper.push_back(
599 V["title"].GetString(),
600 V["type"].GetString(),
601 V["text"].GetString()
602 )
603 );
604
605 }
606 return wrapper;
607 }
627 Song getSong(string songTitle, string artistName = "") {
628 using namespace rapidjson;
629
630 Document d;
631 vector<Song> wrapper;
632 string url = "http://static-data.bridgesuncc.org/api/songs/find/";
633 // retrieve the data and parse
634 if (songTitle.size() > 0)
635 url += ServerComm::encodeURLPart(songTitle);
636 else {
637 throw "Incorrect use of getSong. songTitle should be given.";
638 }
639
640 if (artistName.size() > 0)
641 url += "?artistName=" + ServerComm::encodeURLPart(artistName);
642
643 if (debug())
644 std::cerr << "url: " << url << "\n";
645 d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());
646
647 string artist = (d.HasMember("artist")) ?
648 d["artist"].GetString() : string();
649 string song = (d.HasMember("song")) ?
650 d["song"].GetString() : string();
651 string album = (d.HasMember("album")) ?
652 d["album"].GetString() : string();
653 string lyrics = (d.HasMember("lyrics")) ?
654 d["lyrics"].GetString() : string();
655 string release_date = (d.HasMember("release_date")) ?
656 d["release_date"].GetString() : string();
657
658 return Song (artist, song, album, lyrics, release_date);
659 }
675 vector<Song> getSongData() {
676 using namespace rapidjson;
677
678 Document d;
679 vector<Song> all_songs;
680
681 string url = "http://static-data.bridgesuncc.org/api/songs/";
682 // retrieve the data and parse
683
684 d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());
685
686 const Value& D = d["data"];
687
688 // get the songs and put them into a vector
689 for (SizeType i = 0; i < D.Size(); i++) {
690 const Value& v = D[i];
691
692 //cout << v["artist"].GetString() << endl;
693 string artist = (v.HasMember("artist")) ? v["artist"].GetString() : string();
694 string song = (v.HasMember("song")) ? v["song"].GetString() : string();
695 string album = (v.HasMember("album")) ? v["album"].GetString() : string();
696 string lyrics = (v.HasMember("lyrics")) ? v["lyrics"].GetString() : string();
697 string release_date = (v.HasMember("release_date")) ?
698 v["release_date"].GetString() : string();
699 all_songs.push_back( Song ( artist, song, album, lyrics, release_date) );
700
701 }
702 return all_songs;
703 }
715 private:
716
717 GutenbergBook getAGutenbergBookMetaData(const rapidjson::Value& V) {
718 using namespace rapidjson;
719
720 const string s_id = V["id"].GetString();
721
722 const int id = std::stoi(s_id);
723
724 string title = V["title"].GetString();
725
726 const Value& A = V["authors"];
727 vector<string> authors;
728 for (SizeType j = 0; j < A.Size(); j++)
729 authors.push_back(A[j].GetString());
730
731 const Value& L = V["lang"];
732 string lang = L.GetString();
733
734 const Value& da = V["date_added"];
735 string data_added = da.GetString();
736
737 const Value& G = V["genres"];
738 vector<string> genres;
739 for (SizeType j = 0; j < G.Size(); j++)
740 genres.push_back(G[j].GetString());
741
742 GutenbergBook gbook = GutenbergBook(title, id, authors, lang, genres, data_added);
743
744 return gbook;
745 }
746
747 public:
761 using namespace rapidjson;
762
763 // get the query string to get meta data of book
764 string url = getGutenbergBaseURL() + "/meta?id=" + std::to_string(id);
765
766 // make the query
767 Document d;
768 d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());
769 // only 1 book
770 return getAGutenbergBookMetaData(d["book_list"][0]);
771 }
782 vector<GutenbergBook> getGutenbergBookMetaData(string term, string category) {
783 using namespace rapidjson;
784
785 // get the query string to get meta data of book
786 string url = getGutenbergBaseURL() + "/search?search=" +
787 ServerComm::encodeURLPart(term) + "&type="
788 + ServerComm::encodeURLPart(category);
789 // make the query
790 cout << url << endl;
791 Document d;
792 d.Parse(ServerComm::makeRequest(url, {"Accept: application/json"}).c_str());
793
794 vector<GutenbergBook> book_list;
795 int size = d["book_list"].Size();
796
797 book_list.clear();
798 for (int k = 0; k < size; k++)
799 book_list.push_back(getAGutenbergBookMetaData(d["book_list"][k]));
800
801 return book_list;
802 }
803
811 string getGutenbergBookText(int id = 0) {
812 using namespace rapidjson;
813
814 //URL for data request
815 string data_url = getGutenbergBaseURL() + "/book?id=" + std::to_string(id);
816
817 // generate the hash code - use the id
818 string hash_value = "gutenberg" + std::to_string(id);
819
820 // get the dataset from cache, else from the server
821 string book_data = getDataSetJSON(data_url, hash_value, "gutenberg");
822 // parse, and get the book text
823 Document d;
824 d.Parse(book_data.c_str());
825
826 string s = std::to_string(id);
827 return d[s.c_str()].GetString();
828 }
829
836 vector<CancerIncidence> getCancerIncidenceData(int num = 0) {
837 using namespace rapidjson;
838
839 Document d;
840 vector<CancerIncidence> wrapper;
841 string url = "http://static-data.bridgesuncc.org/api/cancer/withlocations";
842 if (num > 0) {
843 url += "?limit=" + to_string(num);
844 }
845
846 d.Parse(ServerComm::makeRequest( url, {"Accept: application/json"}).c_str());
847
848 // get the JSON dataset
849 const Value& D = d["data"];
850
852 for (SizeType i = 0; i < D.Size(); i++) {
853 const Value& v = D[i];
854 const Value& age = v["Age"];
855
856 c.setAgeAdjustedRate( age["Age Adjusted Rate"].GetDouble());
857 c.setAgeAdjustedCI_Lower(age["Age Adjusted CI Lower"].GetDouble());
858 c.setAgeAdjustedCI_Upper(age["Age Adjusted CI Upper"].GetDouble());
859
860 c.setYear(v["Year"].GetInt());
861
862 const Value& data = v["Data"];
863 c.setCrudeRate(data["Crude Rate"].GetDouble());
864 c.setCrudeRate_CI_Lower(data["Crude CI Lower"].GetDouble());
865 c.setCrudeRate_CI_Upper(data["Crude CI Upper"].GetDouble());
866 c.setRace(data["Race"].GetString());
867 c.setPopulation(data["Population"].GetInt());
868 c.setEventType(data["Event Type"].GetString());
869 c.setCount(data["Count"].GetInt());
870
871 c.setAffectedArea(v["Area"].GetString());
872
873 const Value& loc = v["loc"];
874 c.setLocationX(loc[0].GetDouble());
875 c.setLocationY(loc[1].GetDouble());
876
877 wrapper.push_back(c);
878 }
879 return wrapper;
880 }
888 private:
889 OSMData getOSMDataFromJSON (const string& osm_json) {
890 using namespace rapidjson;
891
892 Document osm_data;
893
894 osm_data.Parse(osm_json.c_str());
895
896 // create an osm data object
897 OSMData osm;
898
899 if (osm_data.HasMember("nodes")) {
900 vector<OSMVertex> vertices;
901 Value& nodes = osm_data["nodes"];
902
903 vector<long> vertex_ids;
904 // get the vertices
905 for (SizeType i = 0; i < nodes.Size(); i++) {
906 const Value& node = nodes[i];
907 OSMVertex::OSMVertexID id = node[0].GetInt64();
908
909 vertex_ids.push_back(id);
910 double lat = node[1].GetDouble(), longit = node[2].GetDouble();
911 vertices.push_back(OSMVertex(id, lat, longit));
912 }
913 osm.setVertices(vertices);
914 }
915 // add vertices to object
916 // get the edges
917
918 if (osm_data.HasMember("edges")) {
919 vector<OSMEdge> edges;
920 Value& links = osm_data["edges"];
921
922 for (SizeType i = 0; i < links.Size(); i++) {
923 const Value& link = links[i];
924 OSMVertex::OSMVertexID id1 = link[0].GetInt64();
925 OSMVertex::OSMVertexID id2 = link[1].GetInt64();
926 double dist = link[2].GetDouble();
927
928 edges.push_back(OSMEdge(id1, id2, dist));
929 }
930 osm.setEdges(edges);
931 }
932 // add edges to object
933
934 if (osm_data.HasMember("meta")) {
935 // get lat long range
936 Value& meta = osm_data["meta"];
937 double lat_min = meta["lat_min"].GetDouble();
938 double lat_max = meta["lat_max"].GetDouble();
939 double longit_min = meta["lon_min"].GetDouble();
940 double longit_max = meta["lon_max"].GetDouble();
941 osm.setLatLongRange(lat_min, lat_max, longit_min, longit_max);
942 // get dataset name
943 osm.setName(meta["name"].GetString());
944 }
945 return osm;
946 }
947
948 public:
949
972 OSMData getOSMData (double lat_min, double long_min,
973 double lat_max, double long_max, string level = "default") {
974
975 //URL for hash request
976 string hash_url = getOSMBaseURL() +
977 "hash?minLon=" + std::to_string(long_min) +
978 "&minLat=" + std::to_string(lat_min) +
979 "&maxLon=" + std::to_string(long_max) +
980 "&maxLat=" + std::to_string(lat_max) +
981 "&level=" + ServerComm::encodeURLPart(level);
982
983 //URL to request map
984 string osm_url =
985 getOSMBaseURL() + "coords?minLon=" + std::to_string(long_min) +
986 "&minLat=" + std::to_string(lat_min) +
987 "&maxLon=" + std::to_string(long_max) +
988 "&maxLat=" + std::to_string(lat_max) +
989 "&level=" + ServerComm::encodeURLPart(level);
990
991 // get the data set from the server or, if available, from
992 // a local cache
993 string osm_json = getDataSetJSON(osm_url, hash_url, "osm");
994
995 //tries to get hash value for bounding box map
996 return getOSMDataFromJSON(osm_json);
997 }
998
1013 vector<Amenity> getAmenityData(double minLat, double minLon, double
1014 maxLat, double maxLon, std::string amenity) {
1015
1016 std::string amenity_url = getOSMBaseURL() + "amenity?minLon=" +
1017 ServerComm::encodeURLPart(std::to_string(minLon)) +
1018 "&minLat=" + ServerComm::encodeURLPart(std::to_string(minLat)) +
1019 "&maxLon=" + ServerComm::encodeURLPart(std::to_string(maxLon)) +
1020 "&maxLat=" + ServerComm::encodeURLPart(std::to_string(maxLat)) +
1021 "&amenity=" + ServerComm::encodeURLPart(amenity);
1022
1023 std::string hash_url = getOSMBaseURL() + "hash?minLon=" +
1024 ServerComm::encodeURLPart(std::to_string(minLon)) +
1025 "&minLat=" + ServerComm::encodeURLPart(std::to_string(minLat)) +
1026 "&maxLon=" + ServerComm::encodeURLPart(std::to_string(maxLon)) +
1027 "&maxLat=" + ServerComm::encodeURLPart(std::to_string(maxLat)) +
1028 "&amenity=" + ServerComm::encodeURLPart(amenity);
1029
1030 // make the query to the server to get a JSON of the amenities
1031 // implements caching to keep local copies
1032 string amenity_json = getDataSetJSON(amenity_url, hash_url, "amenity");
1033
1034 // parse the data and return amenity objects
1035 return parseAmenityData (amenity_json);
1036 }
1037
1049 vector<Amenity> getAmenityData(const std::string& location,
1050 const std::string& amenity) {
1051 std::string amenity_url = getOSMBaseURL() + "amenity?location=" +
1052 ServerComm::encodeURLPart(location) +
1053 "&amenity=" + ServerComm::encodeURLPart(amenity);
1054
1055 std::string hash_url = getOSMBaseURL() + "hash?location=" +
1056 ServerComm::encodeURLPart(location) +
1057 "&amenity=" + ServerComm::encodeURLPart(amenity);
1058
1059 // make the query to the server to get a JSON of the amenities
1060 // implements caching to keep local copies
1061 string amenity_json = getDataSetJSON(amenity_url, hash_url, "amenity");
1062
1063 // parse the data and return amenity objects
1064 return parseAmenityData (amenity_json);
1065 }
1066
1079 vector<Amenity> parseAmenityData(string amenity_json) {
1080 using namespace rapidjson;
1081
1082 vector<Amenity> amenities;
1083 Document amenity_content;
1084
1085 amenity_content.Parse(amenity_json.c_str());
1086 if (amenity_content.HasMember("nodes")) {
1087 const Value& nodes = amenity_content["nodes"];
1088 if (amenity_content.HasMember("meta")) {
1089 const Value& meta = amenity_content["meta"];
1090
1091 // first get the meta data
1092 // amenities.setCount(meta["count"].GetInt64());
1093 // amenities.setMinLat(meta["minlat"].GetDouble());
1094 // amenities.setMinLon(meta["minlon"].GetDouble());
1095 // amenities.setMaxLat(meta["maxlat"].GetDouble());
1096 // amenities.setMaxLon(meta["maxlon"].GetDouble());
1097
1098 Amenity amen;
1099 for (SizeType i = 0; i < nodes.Size(); i++) {
1100 // get amenity data
1101 const Value& node = nodes[i];
1102 amen.setId(node[0].GetInt64());
1103 amen.setLat(node[1].GetDouble());
1104 amen.setLon(node[2].GetDouble());
1105 amen.setName(node[3].GetString());
1106
1107 // add to the list
1108 amenities.push_back(amen);
1109 }
1110 }
1111 else {
1112 cout << "meta data not found!\n";
1113 throw;
1114 }
1115 }
1116 else {
1117 cout << "nodes data not found!\n";
1118 throw;
1119 }
1120 return amenities;
1121 }
1122
1142 OSMData getOSMData (string location, string level = "default") {
1143 //URL for hash request
1144 string hash_url = getOSMBaseURL() + "hash?location=" +
1145 ServerComm::encodeURLPart(location) +
1146 "&level=" + ServerComm::encodeURLPart(level);
1147
1148 //URL to request map
1149 string osm_url = getOSMBaseURL() +
1150 "loc?location=" + ServerComm::encodeURLPart(location) +
1151 "&level=" + ServerComm::encodeURLPart(level);
1152
1153 // get the data set from the server or, if available, from
1154 // a local cache
1155 string osm_json = getDataSetJSON(osm_url, hash_url, "osm");
1156
1157 return getOSMDataFromJSON(osm_json);
1158 }
1159
1176 const std::string& user,
1177 int assignment,
1178 int subassignment = 0) {
1179
1181
1182 std::string s = this->getAssignment(user, assignment, subassignment);
1183
1184 rapidjson::Document doc;
1185 doc.Parse(s.c_str());
1186 if (doc.HasParseError())
1187 throw "Malformed JSON";
1188
1189 //Access doc["assignmentJSON"]
1190 const auto& assjson = doc.FindMember("assignmentJSON");
1191
1192 if (assjson == doc.MemberEnd())
1193 throw "Malformed GraphAdjacencyList JSON: no assignmentJSON";
1194
1195 //Access doc["assignmentJSON"]["data"]
1196 const auto& dataArray = assjson->value.FindMember("data");
1197
1198 if (dataArray == assjson->value.MemberEnd()
1199 || dataArray->value.IsArray() == false)
1200 throw "Malformed GraphAdjacencyList JSON: No data";
1201
1202 const auto& data = dataArray->value.GetArray()[0];
1203
1204 //Access doc["assignmentJSON"]["data"][0]["visual"]
1205 const auto& dataVisual = data.FindMember("visual");
1206
1207 if (dataVisual == data.MemberEnd() ||
1208 dataVisual->value.IsString() == false)
1209 throw "Malformed GraphAdjacencyList JSON";
1210
1211 std::string assignment_type = dataVisual->value.GetString();
1212
1213 if (assignment_type != "GraphAdjacencyList")
1214 throw "Malformed GraphAdjacencyList JSON: Not a GraphAdjacencyList";
1215
1216 //reconstructing vertices out of nodes, and using the optional "name" as the data associated
1217 {
1218 const auto& nodes = data.FindMember("nodes");
1219 if (nodes == data.MemberEnd() ||
1220 nodes->value.IsArray() == false)
1221 throw "Malformed GraphAdjacencyList JSON: malformed nodes";
1222
1223 const auto& nodeArray = nodes->value.GetArray();
1224 int nbVertex = nodeArray.Size();
1225 for (int i = 0; i < nbVertex; ++i) {
1226 std::string name;
1227
1228 const auto& vertexJSONstr = nodeArray[i];
1229
1230 const auto& nameJSON = vertexJSONstr.FindMember("name");
1231 if (nameJSON != vertexJSONstr.MemberEnd()
1232 && nameJSON->value.IsString()) {
1233 name = nameJSON->value.GetString();
1234 }
1235 gr.addVertex(i, name);
1236 }
1237 }
1238
1239 //reconstructing links, and using "label" as data associated with the link
1240 {
1241 const auto& links = data.FindMember("links");
1242 if (links == data.MemberEnd() ||
1243 links->value.IsArray() == false)
1244 throw "Malformed GraphAdjacencyList JSON: malformed links";
1245
1246 const auto& linkArray = links->value.GetArray();
1247 int nbLink = linkArray.Size();
1248 for (int i = 0; i < nbLink; ++i) {
1249 std::string name;
1250 int src;
1251 int dest;
1252 int wgt;
1253
1254 const auto& linkJSONstr = linkArray[i];
1255
1256 //checking label. Maybe does not exist? (is that field optional?)
1257 const auto& nameJSON = linkJSONstr.FindMember("label");
1258 if (nameJSON != linkJSONstr.MemberEnd()
1259 && nameJSON->value.IsString()) {
1260 name = nameJSON->value.GetString();
1261 }
1262
1263 //checking source
1264 const auto& srcJSON = linkJSONstr.FindMember("source");
1265 if (srcJSON == linkJSONstr.MemberEnd()
1266 || srcJSON->value.IsInt() == false) {
1267 throw "Malformed GraphAdjacencyList JSON: malformed link";
1268 }
1269 src = srcJSON->value.GetInt();
1270
1271 //checking destination
1272 const auto& dstJSON = linkJSONstr.FindMember("target");
1273 if (dstJSON == linkJSONstr.MemberEnd()
1274 || dstJSON->value.IsInt() == false) {
1275 throw "Malformed GraphAdjacencyList JSON: malformed link";
1276 }
1277 dest = dstJSON->value.GetInt();
1278
1279 //checking weight. //why is weight a mandatory parameter?
1280 const auto& wgtJSON = linkJSONstr.FindMember("weight");
1281 if (wgtJSON == linkJSONstr.MemberEnd()
1282 || wgtJSON->value.IsInt() == false) {
1283 throw "Malformed GraphAdjacencyList JSON: malformed link";
1284 }
1285 wgt = wgtJSON->value.GetInt();
1286
1287 //adding edge.
1288 gr.addEdge(src, dest, name);
1289 }
1290 }
1291
1292 return gr;
1293 }
1294
1303 int assignment,
1304 int subassignment = 0) {
1305
1306 std::string s = this->getAssignment(user, assignment, subassignment);
1307
1308 rapidjson::Document doc;
1309 doc.Parse(s.c_str());
1310 if (doc.HasParseError())
1311 throw "Malformed JSON";
1312
1313 try {
1314 std::string assignment_type = doc["assignment_type"].GetString();
1315
1316 if (assignment_type != "ColorGrid")
1317 throw "Malformed ColorGrid JSON: Not a ColorGrid";
1318 }
1319 catch (rapidjson_exception re) {
1320 throw "Malformed JSON: Not a Bridges assignment?";
1321 }
1322
1323 try {
1324 auto& data = doc["data"][0];
1325
1326 std::string encoding = data["encoding"].GetString();
1327 if (encoding != "RAW" && encoding != "RLE")
1328 throw "Malformed ColorGrid JSON: encoding not supported";
1329
1330 //Access doc["data"][0]["dimensions"]
1331 const auto& dimensions = data["dimensions"];
1332 int dimx = dimensions[0].GetInt();
1333 int dimy = dimensions[1].GetInt();
1334
1335 if (debug())
1336 std::cerr << "Dimensions: " << dimx << "x" << dimy << std::endl;
1337
1338 //Access doc["data"][0]["nodes"][0]
1339 std::string base64_encoded_assignment = data["nodes"][0].GetString();
1340
1341 std::vector<bridges::BYTE> decoded = bridges::base64::decode(base64_encoded_assignment);
1342
1343 bridges::ColorGrid cg (dimx, dimy);
1344
1345 if (encoding == "RAW") {
1346 if (debug())
1347 std::cerr << "decoding RAW" << std::endl;
1348 if (debug())
1349 std::cerr << "length: " << decoded.size() << std::endl;
1350 if (decoded.size() < dimx * dimy * 4)
1351 throw "Malformed ColorGrid JSON: nodes is smaller than expected";
1352
1353 //first pixel
1354 //std::cerr<<(int)decoded[0]<<" "<<(int)decoded[1]<<" "<<(int)decoded[2]<<" "<<(int)decoded[3]<<std::endl;
1355
1356 //bridges::ColorGrid* ptr = new bridges::ColorGrid (dimx, dimy);
1357
1358 size_t base = 0;
1359
1360 for (int x = 0; x < dimx; ++x) {
1361 for (int y = 0; y < dimy; ++y) {
1362 bridges::Color c ((int)decoded[base],
1363 (int)decoded[base + 1],
1364 (int)decoded[base + 2],
1365 (int)decoded[base + 3]
1366 );
1367
1368 cg.set(x, y, c);
1369 base += 4;
1370 }
1371 }
1372 }
1373 else if (encoding == "RLE") {
1374 if (debug())
1375 std::cerr << "Decoding RLE" << std::endl;
1376
1377 int currentInDecoded = 0;
1378 int currentInCG = 0;
1379 while (currentInDecoded != decoded.size()) {
1380 if (currentInDecoded + 5 > decoded.size())
1381 throw "Malformed ColorGrid JSON: nodes is not a multiple of 5";
1382
1383 int repeat = (BYTE) decoded[currentInDecoded++];
1384 int r = (BYTE) decoded[currentInDecoded++];
1385 int g = (BYTE) decoded[currentInDecoded++];
1386 int b = (BYTE) decoded[currentInDecoded++];
1387 int a = (BYTE) decoded[currentInDecoded++];
1388
1389 if (debug())
1390 std::cerr << "indecoded: " << currentInDecoded
1391 << " repeat: " << (int)repeat
1392 << " color(" << (int)r << "," << (int)g << "," << (int)b << "," << (int)a << ")"
1393 << std::endl;
1394
1395 bridges::Color c (r, g, b, a);
1396
1397 while (repeat >= 0) {
1398 int posX = currentInCG / dimy;
1399 int posY = currentInCG % dimy;
1400 if (posX >= dimx || posY >= dimy) {
1401 if (debug())
1402 std::cerr << posX << " " << dimx << " " << posY << " " << dimy << std::endl;
1403 throw "Malformed ColorGrid JSON: Too much data in nodes";
1404 }
1405 cg.set(posX, posY, c);
1406
1407 currentInCG++;
1408 repeat --;
1409 }
1410 }
1411 if (debug())
1412 std::cerr << "written " << currentInCG << " pixels" << std::endl;
1413 if (currentInCG != dimx * dimy)
1414 throw "Malformed ColorGrid JSON: Not enough data in nodes";
1415 }
1416
1417 return cg;
1418 }
1419 catch (rapidjson_exception re) {
1420 throw "Malformed ColorGrid JSON";
1421 }
1422
1423 }
1424 private:
1425 /***
1426 * This function obtains the JSON representation of a particular subassignment.
1427 *
1428 * @return a string that is the JSON representation of the subassignment as stored by the Bridges server.
1429 * @param user the name of the user who uploaded the assignment
1430 * @param assignment the ID of the assignment to get
1431 * @param subassignment the ID of the subassignment to get
1432 ***/
1433 std::string getAssignment(std::string user,
1434 int assignment,
1435 int subassignment = 0) {
1436 std::vector<std::string> headers;
1437
1438 std::stringstream ss;
1439
1441 if (bridges_inst)
1442 ss << bridges_inst->getServerURL();
1443 else
1444 ss << bridges::Bridges::getDefaultServerURL();
1445 ss << "/assignmentJSON/"
1446 << assignment << ".";
1447 ss << std::setfill('0') << std::setw(2) << subassignment;
1448 ss << "/" << ServerComm::encodeURLPart(user);
1449
1450 std::string url = ss.str();
1451
1452 std::string s = bridges::ServerComm::makeRequest(url, headers);
1453
1454 return s;
1455 }
1456
1457 void removeFirstOccurence (std::string & str, const std::string & toRemove) {
1458 size_t pos = str.find(toRemove);
1459 if (pos != std::string::npos) {
1460 str.erase(pos, toRemove.length());
1461 }
1462 }
1463
1473 void getWikidataActorMovieDirect (int yearbegin, int yearend, std::vector<MovieActorWikidata>& vout) {
1474 std::string codename = "wikidata-actormovie-" + std::to_string(yearbegin) + "-" + std::to_string(yearend);
1475 std::string json;
1476 bool from_cache = false;
1477 try {
1478 if (my_cache.inCache(codename)) {
1479 json = my_cache.getDoc(codename);
1480 from_cache = true;
1481 }
1482 }
1483 catch (CacheException& ce) {
1484 //something went bad trying to access the cache
1485 std::cout << "Exception while reading from cache. Ignoring cache and continue.\n( What was:" << ce.what() << ")" << std::endl;
1486 }
1487
1488 if (!from_cache) {
1489 std::vector<std::string> http_headers;
1490 http_headers.push_back("User-Agent: bridges-cxx"); //wikidata kicks you out if you don't have a useragent
1491 http_headers.push_back("Accept: application/json"); //tell wikidata we are OK with JSON
1492
1493 string url = "https://query.wikidata.org/sparql?";
1494
1495 //Q1860 is "English"
1496 //P364 is "original language of film or TV show"
1497 //P161 is "cast member"
1498 //P577 is "publication date"
1499 //A11424 is "film"
1500 //P31 is "instance of"
1501 // "instance of film" is necessary to filter out tv shows
1502 std::string sparqlquery =
1503 "SELECT ?movie ?movieLabel ?actor ?actorLabel WHERE \
1504{\
1505 ?movie wdt:P31 wd:Q11424.\
1506 ?movie wdt:P161 ?actor.\
1507 ?movie wdt:P364 wd:Q1860.\
1508 ?movie wdt:P577 ?date.\
1509 FILTER(YEAR(?date) >= " + std::to_string(yearbegin) + " && YEAR(?date) <= " + std::to_string(yearend) + ").\
1510 SERVICE wikibase:label { bd:serviceParam wikibase:language \"en\". } \
1511}";
1512 url += "query=" + ServerComm::encodeURLPart(sparqlquery);
1513 url += "&";
1514 url += "format=json";
1515
1516 if (debug()) {
1517 std::cout << "URL: " << url << "\n";
1518 }
1519 // get the Wikidata json
1520 json = ServerComm::makeRequest(url, http_headers);
1521
1522 try {
1523 my_cache.putDoc(codename, json);
1524 }
1525 catch (CacheException& ce) {
1526 //something went bad trying to access the cache
1527 std::cerr << "Exception while storing in cache. Weird but not critical. (What was: " << ce.what() << " )" << std::endl;
1528 }
1529 }
1530
1531 {
1532 using namespace rapidjson;
1533 rapidjson::Document doc;
1534 doc.Parse(json.c_str());
1535 if (doc.HasParseError())
1536 throw "Malformed JSON";
1537
1538 try {
1539 const auto& resultsArray = doc["results"]["bindings"].GetArray();
1540
1541 for (auto& mak_json : resultsArray) {
1543
1544 // all wikidata uri start with "http://www.wikidata.org/entity/"
1545 // so strip it out because it does not help discriminate and
1546 // consume memory and runtime to compare string
1547 std::string actoruri = mak_json["actor"]["value"].GetString();
1548 std::string movieuri = mak_json["movie"]["value"].GetString();
1549 removeFirstOccurence (actoruri, "http://www.wikidata.org/entity/");
1550
1551 removeFirstOccurence (movieuri, "http://www.wikidata.org/entity/");
1552
1553 mak.setActorURI(actoruri);
1554 mak.setMovieURI(movieuri);
1555 mak.setActorName(mak_json["actorLabel"]["value"].GetString());
1556 mak.setMovieName(mak_json["movieLabel"]["value"].GetString());
1557 vout.push_back(mak);
1558 }
1559
1560 }
1561 catch (rapidjson_exception re) {
1562 throw "Malformed JSON: Not from wikidata?";
1563 }
1564 }
1565 }
1566 public:
1567
1575 std::vector<MovieActorWikidata> getWikidataActorMovie (int yearbegin, int yearend) {
1576 //Internally this function get the data year by year. This
1577 //is pretty bad because it hits wikidata the first time
1578 //for multiple years. But it enables to work around
1579 //wikidata's time limit. This also works well because the
1580 //Cache will store each year independently and so without
1581 //redundancy. Though I (Erik) am not completely sure that a
1582 //movie can be appear in different years, for instance it
1583 //can be released in the US in 2005 but in canada in
1584 //2006...
1585
1586 std::vector<MovieActorWikidata> ret;
1587 for (int y = yearbegin; y <= yearend; ++y) {
1588 // cout << "getting year " << y << endl;
1589 getWikidataActorMovieDirect (y, y, ret);
1590 }
1591 return ret;
1592 }
1593
1607 double minLat, double minLon,
1608 double maxLat, double maxLon, double res = 0.0166) {
1609
1610 // set up the elevation data url to get the data, given
1611 // a lat/long bounding box
1612
1613 std::string elev_url = getElevationBaseURL() +
1614 "elevation?minLon=" + ServerComm::encodeURLPart(std::to_string(minLon)) +
1615 "&minLat=" + ServerComm::encodeURLPart(std::to_string(minLat)) +
1616 "&maxLon=" + ServerComm::encodeURLPart(std::to_string(maxLon)) +
1617 "&maxLat=" + ServerComm::encodeURLPart(std::to_string(maxLat)) +
1618 "&resX=" + ServerComm::encodeURLPart(std::to_string(res)) +
1619 "&resY=" + ServerComm::encodeURLPart(std::to_string(res));
1620
1621 if (debug())
1622 cout << "Elevation URL:" << elev_url << "\n";
1623 cout << "Elevation URL:" << elev_url << "\n";
1624
1625 std::string hash_url = getElevationBaseURL() +
1626 "hash?minLon=" + ServerComm::encodeURLPart(std::to_string(minLon)) +
1627 "&minLat=" + ServerComm::encodeURLPart(std::to_string(minLat)) +
1628 "&maxLon=" + ServerComm::encodeURLPart(std::to_string(maxLon)) +
1629 "&maxLat=" + ServerComm::encodeURLPart(std::to_string(maxLat)) +
1630 "&resX=" + ServerComm::encodeURLPart(std::to_string(res)) +
1631 "&resY=" + ServerComm::encodeURLPart(std::to_string(res));
1632
1633 if (debug())
1634 cout << "Hash URL:" << hash_url << "\n";
1635
1636 // get the dataset's JSON from the local cache, if available,
1637 // else from the server
1638
1639 string elev_json = getDataSetJSON(elev_url, hash_url, "elevation"); //Erik says: we call that function but the format ain't JSON somehow.
1640
1641 return parseElevationData(elev_json);
1642 }
1643
1652
1653 // use a string stream to parse the data, which is not really a JSON,
1654 // but raw text
1655 stringstream ss(elev_json);
1656
1657 int rows, cols, elev_val;
1658 double ll_x, ll_y, cell_size;
1659 string tmp;
1660
1661 // get the dimensions, origin
1662 ss >> tmp >> cols >> tmp >> rows >>
1663 tmp >> ll_x >> tmp >> ll_y >>
1664 tmp >> cell_size;
1665
1666 if (!ss)
1667 throw "Parse Error";
1668
1669 // create the elevation object
1670 ElevationData elev_data (rows, cols);
1671 elev_data.setxll(ll_x);
1672 elev_data.setyll(ll_y);
1673 elev_data.setCellSize(cell_size);
1674
1675 // load the elevation data
1676 for (int i = 0; i < rows; i++) {
1677 for (int j = 0; j < cols; j++) {
1678 ss >> elev_val;
1679 elev_data.setVal(i, j, elev_val);
1680 }
1681 }
1682 if (!ss)
1683 throw "Parse Error";
1684
1685 return elev_data;
1686 }
1687
1694 std::vector<std::string> getAvailableSubreddits() {
1695 string base_url = getRedditURL();
1696 string url = base_url + "/listJSON";
1697 if (debug()) {
1698 std::cout << "hitting url: " << url << "\n";
1699 }
1700 using namespace rapidjson;
1701 Document doc;
1702 {
1703 std::string s = ServerComm::makeRequest(url, {"Accept: application/json"});
1704 if (debug()) {
1705 std::cout << "Returned JSON:" << s << "\n";
1706 }
1707 try {
1708 doc.Parse(s.c_str());
1709 }
1710 catch (rapidjson_exception& re) {
1711 std::cerr << "malformed subreddit list" << "\n";
1712 std::cerr << "Original exception: " << (std::string)re << "\n";
1713 }
1714 }
1715
1716 std::vector<std::string> subreddits;
1717 try {
1718 for (auto& m : doc.GetArray()) {
1719
1720 std::string subred = m.GetString();
1721 subreddits.push_back(subred);
1722
1723 }
1724 }
1725 catch (rapidjson_exception& re) {
1726 std::cerr << "malformed subreddit list" << "\n";
1727 std::cerr << "Original exception: " << (std::string)re << "\n";
1728 }
1729
1730 return subreddits;
1731
1732 }
1733
1744 vector<Reddit> getRedditData(string subreddit, int time_request = -9999) {
1745 string base_url = getRedditURL();
1746 if (debug()) {
1747 cout << "reddit base url:" << base_url << "\n";
1748 }
1749 string url = base_url + "/cache?subreddit=" + ServerComm::encodeURLPart(subreddit) +
1750 "&time_request=" + std::to_string(time_request);
1751
1752 if (debug()) {
1753 cout << "reddit url:" << url << "\n";
1754 }
1755
1756 using namespace rapidjson;
1757 Document doc;
1758 {
1759 std::string s = ServerComm::makeRequest(url, {"Accept: application/json"});
1760 if (debug()) {
1761 std::cout << "Returned JSON:" << s << "\n";
1762 }
1763 doc.Parse(s.c_str());
1764 }
1765
1766 vector<Reddit> reddit_posts;
1767 for (auto& m : doc.GetObject()) {
1768 try {
1769 if (debug()) {
1770 std::cout << m.name.GetString() << "\n";
1771 }
1772 auto& postJSON = m.value;
1773
1774 std::string id = postJSON["id"].GetString();
1775 std::string title = postJSON["title"].GetString();
1776 std::string author = postJSON["author"].GetString();
1777 int score = postJSON["score"].GetInt();
1778 float vote_ratio = postJSON["vote_ratio"].GetDouble();
1779 int comment_count = postJSON["comment_count"].GetInt();
1780 std::string subreddit = postJSON["subreddit"].GetString();
1781 int posttime = postJSON["post_time"].GetDouble();
1782 std::string url = postJSON["url"].GetString();
1783 std::string text = postJSON["text"].GetString();
1784
1785 Reddit r;
1786 r.setID(id);
1787 r.setTitle(title);
1788 r.setAuthor(author);
1789 r.setScore(score);
1790 r.setVoteRatio(vote_ratio);
1791 r.setCommentCount(comment_count);
1792 r.setSubreddit(subreddit);
1793 r.setPostTime(posttime);
1794 r.setURL(url);
1795 r.setText(text);
1796 reddit_posts.push_back(r);
1797 }
1798 catch (rapidjson_exception& re) {
1799 std::cerr << "malformed Reddit post" << "\n";
1800 std::cerr << "Original exception: " << (std::string)re << "\n";
1801 }
1802 }
1803
1804 return reddit_posts;
1805 }
1806
1807 private:
1817 string getHashCode (string hash_url, string data_type) {
1818 string hash_value;
1819 if (data_type == "osm" || data_type == "amenity" ||
1820 data_type == "elevation") {
1821 hash_value = ServerComm::makeRequest(hash_url, {"Accept: application/json"});
1822 }
1823 else if (data_type == "gutenberg")
1824 hash_value = hash_url;
1825
1826 return hash_value;
1827 }
1828
1850 std::string getDataSetJSON(std::string data_url, std::string hash_url,
1851 std::string data_type) {
1852
1853 std::string data_json = "";
1854
1855 // First check to see if the requested data is stored in local cache
1856 // get hash value for elevation data
1857 if (debug())
1858 cerr << "Checking the cache: Hash url: " << hash_url << "\n";
1859
1860 // generate the hash code
1861 string hash_value = getHashCode(hash_url, data_type);
1862
1863 bool dataloaded = false;
1864
1865 if ((hash_value != "false") && (my_cache.inCache(hash_value) == true)) { //local cache contains the dataset
1866 try {
1867 data_json = my_cache.getDoc(hash_value);
1868 dataloaded = true;
1869 }
1870 catch (CacheException& ce) {
1871 //something went bad trying to access the data in the local cache
1872 std::cout << "Exception while reading from cache. "
1873 << "Ignoring cache and continuing..\n (What was:" << ce.what() << ")\n";
1874 }
1875 }
1876 if (!dataloaded) {
1877 //Data could not get accessed from cache for some reason.
1878 //So teh data need to be access from the remote server.
1879 //Then we will store it in a local cache for future usage.
1880 if (debug())
1881 std::cerr << "Hitting data URL: " << data_url << "\n";
1882
1883 //Requests the data
1884 data_json = ServerComm::makeRequest(data_url,
1885 {"Accept: application/json"});
1886
1887 //Store the data in cache for future reuse
1888 try {
1889 // We need the data's hash code to know where to store it in local cache.
1890 // We may already have it from the previous query.
1891 if (hash_value == "false") {
1892 if (debug())
1893 std::cerr << "Hitting hash URL: " << hash_value << "\n";
1894
1895 hash_value = getHashCode(hash_url, data_type);
1896 }
1897
1898 // This test should only ever be true if something wrong happens server-side
1899 if (hash_value == "false") {
1900 std::cerr << "Error while gathering hash value for " << data_type << " dataset..\n"
1901 << "Weird but not critical.\n";
1902 }
1903 else {
1904 my_cache.putDoc(hash_value, data_json);
1905 }
1906 }
1907 catch (CacheException& ce) {
1908 //something went bad trying to access the cache
1909 std::cerr << "Exception while storing in cache. " <<
1910 "Weird but not critical.\n" <<
1911 "(What was: " << ce.what() << ")\n";
1912 if (debug())
1913 std::cerr << "Tried to store hash=" << hash_value <<
1914 " key = " << data_json << std::endl;
1915 }
1916 }
1917
1918 if (debug())
1919 cout << data_json;
1920
1921 return data_json;
1922 }
1923
1924 }; // class DataSource
1925} // namespace bridges
1926#endif
const vector< string > all_us_states
Definition: MapConstants.h:14
const vector< string > all_countries
Definition: MapConstants.h:27
Class that hold Open Street Map Amenity data.
Definition: Amenity.h:22
void setLat(double latitude)
Definition: Amenity.h:90
void setId(long id)
Definition: Amenity.h:73
void setLon(double longitude)
Definition: Amenity.h:108
void setName(string n)
Definition: Amenity.h:124
This class contains methods to connect and transmit a user's data structure representation to the Bri...
Definition: Bridges.h:51
Definition: Cache.h:17
virtual const char * what() const noexcept
Definition: Cache.h:26
This class provides an API to various data sources used in BRIDGES.
Definition: DataSource.h:69
vector< ActorMovieIMDB > getActorMovieIMDBData2()
Get ActorMovie IMDB Data Data is retrieved, formatted into a list of ActorMovieIMDB objects.
Definition: DataSource.h:494
vector< Amenity > getAmenityData(double minLat, double minLon, double maxLat, double maxLon, std::string amenity)
Definition: DataSource.h:1013
DataSource(bridges::Bridges &br)
Definition: DataSource.h:147
vector< City > getUSCities(unordered_map< string, string > params)
Retrieves US city data based on a set of filtering parameters.
Definition: DataSource.h:190
vector< USState > getUSMapCountyData(vector< string > state_names, bool view_counties=true)
Get US State boundaries and counties of specified states.
Definition: DataSource.h:299
OSMData getOSMData(string location, string level="default")
Definition: DataSource.h:1142
vector< Song > getSongData()
Get data of the songs (including lyrics) using the Genius API https://docs.genius....
Definition: DataSource.h:675
std::vector< MovieActorWikidata > getWikidataActorMovie(int yearbegin, int yearend)
This function returns the Movie and Actors playing in them between two years.
Definition: DataSource.h:1575
vector< USState > getUSMapCountyData()
Get US State boundaries and counties af all 50 states.
Definition: DataSource.h:287
string getGutenbergBookText(int id=0)
Get the full text of the book with the provided id.
Definition: DataSource.h:811
vector< CancerIncidence > getCancerIncidenceData(int num=0)
Retrieves the CDC dataset of Cancer Incidence. Data is retrieved into a vector of records See CancerI...
Definition: DataSource.h:836
vector< GutenbergBook > getGutenbergBookMetaData(string term, string category)
Search the gutenberg data for retrieving meta data of books matching a string and a category.
Definition: DataSource.h:782
OSMData getOSMData(double lat_min, double long_min, double lat_max, double long_max, string level="default")
Get OpenStreetMap data given a bounding rectangle of lat/long values.
Definition: DataSource.h:972
vector< Amenity > parseAmenityData(string amenity_json)
Parses the amenity string and returns an AmenityData object.
Definition: DataSource.h:1079
vector< Shakespeare > getShakespeareData(string type="", bool textonly=false)
Get data of Shakespeare works (plays, poems)
Definition: DataSource.h:578
Song getSong(string songTitle, string artistName="")
Get data of a particular songs (including lyrics) using the Genius API (https://docs....
Definition: DataSource.h:627
bridges::GraphAdjList< int, std::string > getGraphFromAssignment(const std::string &user, int assignment, int subassignment=0)
Definition: DataSource.h:1175
ElevationData parseElevationData(string elev_json)
Parses the elevation data string and retuns an Elevation object.
Definition: DataSource.h:1651
vector< Reddit > getRedditData(string subreddit, int time_request=-9999)
retrieves the reddit posts from a subreddit.
Definition: DataSource.h:1744
vector< Game > getGameData()
Get meta data of the IGN games collection.
Definition: DataSource.h:423
vector< Amenity > getAmenityData(const std::string &location, const std::string &amenity)
Definition: DataSource.h:1049
bridges::ColorGrid getColorGridFromAssignment(const std::string &user, int assignment, int subassignment=0)
Definition: DataSource.h:1302
void setSourceType(string type)
set data server type
Definition: DataSource.h:159
ElevationData getElevationData(double minLat, double minLon, double maxLat, double maxLon, double res=0.0166)
Definition: DataSource.h:1606
vector< ActorMovieIMDB > getActorMovieIMDBData(int number=0)
Get ActorMovie IMDB Data Data is retrieved, formatted into a list of ActorMovieIMDB objects.
Definition: DataSource.h:461
vector< Country > getWorldMapData(vector< string > countries)
Definition: DataSource.h:368
vector< USState > getUSMapData()
Get US State data of all 50 states.
Definition: DataSource.h:277
vector< EarthquakeUSGS > getEarthquakeUSGSData(int number=0)
Get USGS earthquake data.
Definition: DataSource.h:532
vector< Country > getWorldMapData()
Definition: DataSource.h:365
std::vector< std::string > getAvailableSubreddits()
retrieves the subreddits made available by BRIDGES
Definition: DataSource.h:1694
DataSource(bridges::Bridges *br=nullptr)
Definition: DataSource.h:142
GutenbergBook getGutenbergBookMetaData(int id=0)
Get meta data of a single Gutenberg book This function retrieves, and formats the data into a list of...
Definition: DataSource.h:760
A class to hold actor movie data – using IMDB dataset.
Definition: ActorMovieIMDB.h:31
A class representing the attributes for cancer incidence.
Definition: CancerIncidence.h:32
Class that holds data of a city.
Definition: City.h:29
Definition: Country.h:25
Class that hold earthquake data, for use with USGIS retrieved quake data.
Definition: EarthquakeUSGS.h:28
Class that hold elevation data.
Definition: ElevationData.h:27
void setyll(int y_ll)
Definition: ElevationData.h:206
void setxll(int x_ll)
Definition: ElevationData.h:189
void setVal(int r, int c, int val)
Definition: ElevationData.h:151
void setCellSize(float cell_size)
Definition: ElevationData.h:224
A Game object, used along with the Games data source.
Definition: Game.h:34
A Gutenberg Book object (meta data and book's full text)
Definition: GutenbergBook.h:32
This is a helper class for accessing actor-movie data from Wikidata.
Definition: MovieActorWikidata.h:23
void setMovieURI(std::string mu)
Definition: MovieActorWikidata.h:40
void setActorName(std::string an)
Definition: MovieActorWikidata.h:64
void setActorURI(std::string au)
Definition: MovieActorWikidata.h:48
void setMovieName(std::string mn)
Definition: MovieActorWikidata.h:56
Class that hold Open Street Map Data.
Definition: OSMData.h:38
void setEdges(const vector< OSMEdge > &e)
set edges
Definition: OSMData.h:307
void setVertices(const vector< OSMVertex > &verts)
replace the vertices stored by this new set.
Definition: OSMData.h:269
void setName(const string &n)
change the name of the dataset
Definition: OSMData.h:224
void setLatLongRange(double *lat_range, double *longit_range)
set the latitude and longitude range of the dataset
Definition: OSMData.h:111
Class that hold Open Street Map edges.
Definition: OSMEdge.h:27
Class that hold Open Street Map vertices.
Definition: OSMVertex.h:33
long OSMVertexID
Definition: OSMVertex.h:35
An object to represent a Reddit post, used along with the Reddit data source.
Definition: Reddit.h:28
A Shakespeare Data source object containing sonnets, poems and plays.
Definition: Shakespeare.h:31
A Song object, used along with the Songs data source.
Definition: Song.h:27
This object contains US county related information.
Definition: USCounty.h:26
Definition: USState.h:25
This is a class in BRIDGES for representing an image.
Definition: ColorGrid.h:22
This class represents Color, and supports rgba, hexadecimal and named color values.
Definition: Color.h:50
This class provides methods to represent adjacency list based graphs.
Definition: GraphAdjList.h:110
void addVertex(const K &k, const E1 &e=E1())
Adds a vertex to the graph.
Definition: GraphAdjList.h:175
void addEdge(const K &src, const K &dest, const E2 &data=E2())
Definition: GraphAdjList.h:198
void set(int row, int col, E val)
Set the grid value for the (row, col) element.
Definition: Grid.h:186
Definition: Cache.h:201
virtual void putDoc(const std::string &hash_value, const std::string &content) override
Definition: Cache.h:229
virtual bool inCache(const std::string &hash_value) override
Definition: Cache.h:222
virtual std::string getDoc(const std::string &hash_value) override
Definition: Cache.h:213
vector< BYTE > decode(string const &encoded_string)
Definition: base64.h:100
Definition: ActorMovieIMDB.h:10
Definition: Array.h:9
these methods convert byte arrays in to base64 codes and are used in BRIDGES to represent the color a...
Definition: alltypes.h:4
unsigned char BYTE
Definition: base64.h:43
Definition: JSONutil.h:6