Browse Source

Wait between api calls, fixes for html bytes

Ubuntu 3 years ago
parent
commit
b14b5c64c9
1 changed files with 8 additions and 4 deletions
  1. 8 4
      getusers.py

+ 8 - 4
getusers.py

@@ -5,12 +5,14 @@ Script to get data from motomirko and geocode it.
5 5
 import urllib.request
6 6
 import urllib.parse
7 7
 import json
8
+import time
8 9
 
9
-DEBUG = False
10
+DEBUG = True 
10 11
 
11 12
 API_URL = "https://motomirko.pl/"
12 13
 API_KEY = "ciach"
13 14
 API_USER = "mirkomapa"
15
+API_SLEEP = 1
14 16
 
15 17
 def debug(text):
16 18
     """Debug messages"""
@@ -46,7 +48,7 @@ def get_users():
46 48
     while len(users) < total_users - non_active:
47 49
         url = discourse_api_url(next_page.replace("directory_items?", "directory_items.json?"))
48 50
         debug("In get_users: {0}".format(url))
49
-        response = discourse_api(url).read()
51
+        response = discourse_api(url).read().decode('utf-8')
50 52
         response = json.loads(response)
51 53
         total_users = response["total_rows_directory_items"]
52 54
         next_page = response["load_more_directory_items"]
@@ -79,7 +81,7 @@ def get_latlon(location):
79 81
         print("Geocoding URL error: {0}".format(e.reason))
80 82
         exit(4)
81 83
 
82
-    response = json.loads(response.read())
84
+    response = json.loads(response.read().decode('utf-8'))
83 85
 
84 86
     return response
85 87
 
@@ -92,7 +94,7 @@ def get_data(users):
92 94
 
93 95
         data = dict()
94 96
         url = discourse_api_url("users/{0}.json?".format(user))
95
-        user_data = discourse_api(url).read()
97
+        user_data = discourse_api(url).read().decode('utf-8')
96 98
         user_data = json.loads(user_data)
97 99
 
98 100
         if "location" in user_data["user"]:
@@ -112,6 +114,8 @@ def get_data(users):
112 114
             else:
113 115
                 debug("In get_data, no location for {0}".format(user))
114 116
 
117
+        time.sleep(API_SLEEP)
118
+
115 119
     return users_data
116 120
 
117 121
 if __name__ == "__main__":