X-Git-Url: http://average.org/gitweb/?a=blobdiff_plain;f=gps303%2Focid_dload.py;fp=gps303%2Focid_dload.py;h=0000000000000000000000000000000000000000;hb=dbdf9d63af31770ad57302e16b17a2fdc526773f;hp=dfd013f9dbed519c3c21b49a485af218402275cd;hpb=bf48ccad4b4b91e7d7e09d1087f5953bc2db97d7;p=loctrkd.git

diff --git a/gps303/ocid_dload.py b/gps303/ocid_dload.py
deleted file mode 100644
index dfd013f..0000000
--- a/gps303/ocid_dload.py
+++ /dev/null
@@ -1,135 +0,0 @@
-from configparser import ConfigParser, NoOptionError
-import csv
-from logging import getLogger
-import requests
-from sqlite3 import connect
-from typing import Any, IO, Optional
-from zlib import decompressobj, MAX_WBITS
-
-from . import common
-
-log = getLogger("gps303/ocid_dload")
-
-RURL = (
-    "https://opencellid.org/ocid/downloads"
-    "?token={token}&type={dltype}&file={fname}.csv.gz"
-)
-
-SCHEMA = """create table if not exists cells (
-  "radio" text,
-  "mcc" int,
-  "net" int,
-  "area" int,
-  "cell" int,
-  "unit" int,
-  "lon" int,
-  "lat" int,
-  "range" int,
-  "samples" int,
-  "changeable" int,
-  "created" int,
-  "updated" int,
-  "averageSignal" int
-)"""
-DBINDEX = "create index if not exists cell_idx on cells (area, cell)"
-
-
-class unzipped:
-    """
-    File-like object that unzips http response body.
-    read(size) method returns chunks of binary data as bytes
-    When used as iterator, splits data to lines
-    and yelds them as strings.
-    """
-
-    def __init__(self, zstream: IO[bytes]) -> None:
-        self.zstream = zstream
-        self.decoder: Optional[Any] = decompressobj(16 + MAX_WBITS)
-        self.outdata = b""
-        self.line = b""
-
-    def read(self, n: int = 1024) -> bytes:
-        if self.decoder is None:
-            return b""
-        while len(self.outdata) < n:
-            raw_data = self.zstream.read(n)
-            self.outdata += self.decoder.decompress(raw_data)
-            if not raw_data:
-                self.decoder = None
-                break
-        if self.outdata:
-            data, self.outdata = self.outdata[:n], self.outdata[n:]
-            return data
-        return b""
-
-    def __next__(self) -> str:
-        while True:
-            splittry = self.line.split(b"\n", maxsplit=1)
-            if len(splittry) > 1:
-                break
-            moredata = self.read(256)
-            if not moredata:
-                raise StopIteration
-            self.line += moredata
-        line, rest = splittry
-        self.line = rest
-        return line.decode("utf-8")
-
-    def __iter__(self) -> "unzipped":
-        return self
-
-
-def main(conf: ConfigParser) -> None:
-    try:
-        url = conf.get("opencellid", "downloadurl")
-        mcc = "<unspecified>"
-    except NoOptionError:
-        try:
-            with open(
-                conf.get("opencellid", "downloadtoken"), encoding="ascii"
-            ) as fl:
-                token = fl.read().strip()
-        except FileNotFoundError:
-            log.warning(
-                "Opencellid access token not configured, cannot download"
-            )
-            return
-        mcc = conf.get("opencellid", "downloadmcc")
-        if mcc == "full":
-            dltype = "full"
-            fname = "cell_towers"
-        else:
-            dltype = "mcc"
-            fname = mcc
-        url = RURL.format(token=token, dltype="mcc", fname=mcc)
-    dbfn = conf.get("opencellid", "dbfn")
-    count = 0
-    with requests.get(url, stream=True) as resp, connect(dbfn) as db:
-        log.debug("Requested %s, result %s", url, resp)
-        if resp.status_code != 200:
-            log.error("Error getting %s: %s", url, resp)
-            return
-        db.execute("pragma journal_mode = wal")
-        db.execute(SCHEMA)
-        db.execute("delete from cells")
-        rows = csv.reader(unzipped(resp.raw))
-        for row in rows:
-            db.execute(
-                """insert into cells
-                   values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
-                row,
-            )
-            count += 1
-        if count < 1:
-            db.rollback()
-            log.warning("Did not get any data for MCC %s, rollback", mcc)
-        else:
-            db.execute(DBINDEX)
-            db.commit()
-            log.info(
-                "repopulated %s with %d records for MCC %s", dbfn, count, mcc
-            )
-
-
-if __name__.endswith("__main__"):
-    main(common.init(log))