1 from configparser import NoOptionError
3 from logging import getLogger
5 from sqlite3 import connect
6 from zlib import decompressobj, MAX_WBITS
10 log = getLogger("gps303/ocid_dload")
13 "https://opencellid.org/ocid/downloads"
14 "?token={token}&type={dltype}&file={fname}.csv.gz"
17 SCHEMA = """create table if not exists cells (
33 DBINDEX = "create index if not exists cell_idx on cells (area, cell)"
38 File-like object that unzips http response body.
39 read(size) method returns chunks of binary data as bytes
40 When used as iterator, splits data to lines
41 and yelds them as strings.
44 def __init__(self, zstream):
45 self.zstream = zstream
46 self.decoder = decompressobj(16 + MAX_WBITS)
50 def read(self, n=None):
51 if self.decoder is None:
53 while len(self.outdata) < n:
54 raw_data = self.zstream.read(n)
55 self.outdata += self.decoder.decompress(raw_data)
60 data, self.outdata = self.outdata[:n], self.outdata[n:]
66 splittry = self.line.split(b"\n", maxsplit=1)
69 moredata = self.read(256)
75 return line.decode("utf-8")
83 url = conf.get("opencellid", "downloadurl")
88 conf.get("opencellid", "downloadtoken"), encoding="ascii"
90 token = fl.read().strip()
91 except FileNotFoundError:
93 "Opencellid access token not configured, cannot download"
96 mcc = conf.get("opencellid", "downloadmcc")
103 url = RURL.format(token=token, dltype="mcc", fname=mcc)
104 dbfn = conf.get("opencellid", "dbfn")
106 with requests.get(url, stream=True) as resp, connect(dbfn) as db:
107 log.debug("Requested %s, result %s", url, resp)
108 if resp.status_code != 200:
109 log.error("Error getting %s: %s", url, resp)
111 db.execute("pragma journal_mode = wal")
113 db.execute("delete from cells")
114 rows = csv.reader(unzipped(resp.raw))
118 values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
124 log.warning("Did not get any data for MCC %s, rollback", mcc)
129 "repopulated %s with %d records for MCC %s", dbfn, count, mcc
133 if __name__.endswith("__main__"):
134 main(common.init(log))