2 from logging import getLogger
4 from sqlite3 import connect
5 from zlib import decompressobj, MAX_WBITS
9 log = getLogger("gps303/ocid_dload")
12 "https://opencellid.org/ocid/downloads"
13 "?token={token}&type={type}&file={mcc}.csv.gz"
16 SCHEMA = """create table if not exists cells (
32 DBINDEX = "create index if not exists cell_idx on cells (area, cell)"
37 File-like object that unzips http response body.
38 read(size) method returns chunks of binary data as bytes
39 When used as iterator, splits data to lines
40 and yelds them as strings.
43 def __init__(self, zstream):
44 self.zstream = zstream
45 self.decoder = decompressobj(16 + MAX_WBITS)
49 def read(self, n=None):
50 if self.decoder is None:
52 while len(self.outdata) < n:
53 raw_data = self.zstream.read(n)
54 self.outdata += self.decoder.decompress(raw_data)
59 data, self.outdata = self.outdata[:n], self.outdata[n:]
65 splittry = self.line.split(b"\n", maxsplit=1)
68 moredata = self.read(256)
74 return line.decode("utf-8")
83 conf.get("opencellid", "downloadtoken"), encoding="ascii"
85 token = fl.read().strip()
86 except FileNotFoundError:
87 log.warning("Opencellid access token not configured, cannot download")
90 mcc = conf.get("opencellid", "downloadmcc")
91 url = RURL.format(token=token, type="mcc", mcc=mcc)
92 # url = "http://localhost:8000/262.csv.gz" # TESTING
93 dbfn = conf.get("opencellid", "dbfn")
95 with requests.get(url, stream=True) as resp, connect(dbfn) as db:
96 log.debug("Requested %s, result %s", url, resp)
97 if resp.status_code != 200:
98 log.error("Error getting %s: %s", url, resp)
100 db.execute("pragma journal_mode = wal")
102 db.execute("delete from cells")
103 rows = csv.reader(unzipped(resp.raw))
107 values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
112 log.info("repopulated %s with %d records for MCC %s", dbfn, count, mcc)
115 if __name__.endswith("__main__"):
116 main(common.init(log))