Do unit test properly.

doc/rest_api.md

@@ -73,7 +73,7 @@ MIME type: `application/json`
 
 Content: JSON object with the following keys:
 
-- `lang`: Language code as given by the `/languages` endpoint. 
+- `lang`: Language code as given by the `/languages` endpoint.
 - `text`: Input text to be transliterated.
 - `capitalize`: One of `first` (capitalize the first letter of the input),
   `all` (capitalize all words separated by spaces), or null (default: apply no

scriptshifter/__init__.py

@@ -15,7 +15,7 @@ SQLite database path.
 This DB stores all the runtime transliteration data.
 """
 DB_PATH = environ.get(
-        "DB_PATH", path.join(APP_ROOT, "data", "scriptshifter.db"))
+        "TXL_DB_PATH", path.join(APP_ROOT, "data", "scriptshifter.db"))
 
 """
 SMTP server for sending email. For a dummy server that just echoes the

scriptshifter/tables/__init__.py

@@ -28,9 +28,6 @@ runtime.
 """
 
 
-TMP_DB_PATH = path.join(
-        path.dirname(DB_PATH), "~tmp." + path.basename(DB_PATH))
-
 DEFAULT_TABLE_DIR = path.join(path.dirname(path.realpath(__file__)), "data")
 # Can be overridden for tests.
 TABLE_DIR = environ.get("TXL_CONFIG_TABLE_DIR", DEFAULT_TABLE_DIR)
@@ -152,6 +149,8 @@ def init_db():
     # Create parent diretories if necessary.
     # If the DB already exists, it will be overwritten ONLY on success at
     # this point.
+    TMP_DB_PATH = path.join(
+            path.dirname(DB_PATH), "~tmp." + path.basename(DB_PATH))
     if path.isfile(TMP_DB_PATH):
         # Remove previous temp file (possibly from failed attempt)
         unlink(TMP_DB_PATH)
@@ -176,6 +175,7 @@ def init_db():
         # If the DB already exists, it will be overwritten ONLY on success at
         # thhis point.
         move(TMP_DB_PATH, DB_PATH)
+        logger.info(f"Database initialized at {DB_PATH}.")
     finally:
         conn.close()
         if path.isfile(TMP_DB_PATH):
@@ -520,6 +520,10 @@ def get_language(lang):
             if len(s2r_hooks):
                 data["script_to_roman"]["hooks"] = s2r_hooks
 
+            double_cap = get_lang_dcap(conn, lang_id)
+            if len(double_cap):
+                data["script_to_roman"]["double_cap"] = double_cap
+
         # Roman to script map, ignore list, and hooks.
 
         if data["has_r2s"]:
@@ -541,10 +545,6 @@ def get_language(lang):
         if len(opt_data):
             data["options"] = opt_data
 
-        double_cap = get_lang_dcap(conn, lang_id)
-        if len(double_cap):
-            data["double_cap"] = double_cap
-
     conn.close()
 
     return data
@@ -652,7 +652,7 @@ def get_lang_hooks(conn, lang_id, t_dir):
             }
         )
 
-    return hooks
+    return dict(hooks)
 
 
 def get_lang_dcap(conn, lang_id):

tests/__init__.py

@@ -1,85 +1,17 @@
-from csv import reader
-from difflib import ndiff
-from glob import glob
 from importlib import reload
-from json import loads as jloads
-from logging import getLogger
-from os import environ, path
+from os import path, environ
+from tempfile import gettempdir
 
-from scriptshifter.trans import transliterate
+import scriptshifter
+from scriptshifter import tables
 
 
 TEST_DIR = path.dirname(path.realpath(__file__))
 TEST_DATA_DIR = path.join(TEST_DIR, "data")
+TEST_CONFIG_DIR = path.join(TEST_DIR, "tables", "data")
 
-logger = getLogger(__name__)
-
-
-def reload_tables():
-    if "TXL_CONFIG_TABLE_DIR" in environ:
-        del environ["TXL_CONFIG_TABLE_DIR"]
-
-    # import here to set modified test config dir.
-    from scriptshifter import tables
-
-    tables.init_db()
-
-    for fname in glob(path.join(TEST_DATA_DIR, "config", ".yml")):
-        tname = path.splitext(path.basename(filename))[1]
-        with tables.get_connection() as conn:
-            tables.populate_table(conn, tname, {"name": fname})
-
-
-    tables.list_tables.cache_clear()
-    tables.get_language.cache_clear()
-    tables.get_lang_map.cache_clear()
-
-    return tables
-
-
-def test_sample(dset):
-    """
-    Test an individual sample set and produce a human-readable report.
-
-    Used outside of automated tests.
-
-    @param dset (str): sample set name (without the .csv extension) found in
-    the `data/script_samples` directory.
-    """
-    deltas = []
-    dset_fpath = path.join(TEST_DATA_DIR, "script_samples", dset + ".csv")
-    log_fpath = path.join(TEST_DATA_DIR, f"test_{dset}.log")
-
-    with open(dset_fpath, newline="") as fh:
-        csv = reader(fh)
-        i = 1
-        for row in csv:
-            logger.info(f"CSV row #{i}")
-            i += 1
-            lang, script, rom = row[:3]
-            if not lang:
-                continue
-            opts = jloads(row[3]) if len(row) > 3 and row[3] else {}
-            trans, warnings = transliterate(
-                    script, lang, t_dir="s2r",
-                    capitalize=opts.get("capitalize"), options=opts)
-            if (trans == rom):
-                print(".", end="")
-            else:
-                print("F", end="")
-                deltas.append((lang, script, ndiff([trans], [rom])))
-
-    with open(log_fpath, "w") as fh:
-        # If no deltas, just truncate the file.
-        for lang, script, delta in deltas:
-            fh.write(f"Language: {lang}\n")
-            fh.write(f"Original: {script}\nDiff (result vs. expected):\n")
-            for dline in delta:
-                fh.write(dline.strip() + "\n")
-            fh.write("\n\n")
-
-    ct = len(deltas)
-    if ct > 0:
-        print(f"{ct} failed tests. See report at {log_fpath}")
-    else:
-        print("All tests passed.")
+# Reload main SS modules after changing environment variables.
+environ["TXL_DB_PATH"] = path.join(gettempdir(), "scriptshifter_unittest.db")
+reload(scriptshifter)
+environ["TXL_CONFIG_TABLE_DIR"] = TEST_CONFIG_DIR
+reload(tables)

tests/data/config/index.yml

@@ -1,6 +0,0 @@
-inherited:
-  name: Test inherited table
-ordering:
-  name: Test ordering
-rot3:
-  name: Test ROT3 hooks

tests/data/script_samples/unittest.csv

@@ -1,9 +1,7 @@
-chinese,從易經解維摩詰經，臺北市大塊文化出版股份有限公司。,"cong yi jing jie wei mo jie jing, Taibei Shi da kuai wen hua chu ban gu fen you xian gong si.",,
-chinese,廖忠俊. 著名狀元榜眼探花傳略,liao zhong jun. zhu ming zhuang yuan bang yan tan hua zhuan lüe,,
-chinese,文學革命論 / 陳獨秀 -- 人的文學 / 周作人 -- 新文學運動的意義 / 張我軍.,wen xue ge ming lun / chen du xiu -- ren de wen xue / zhou zuo ren -- xin wen xue yun dong de yi yi / zhang wo jun.,,
-belarusian,Пётр Клімук : жыццё і подзвіг касманаўта,Pi︠o︡tr Klimuk : z︠h︡ytstsi︠o︡ i podzvih kasmanaŭta,,
-greek_classical,Ἡσιόδου τοῦ Ἀσκραίου Ἔργα καὶ ἡμέραι,Hēsiodou tou Askraiou Erga kai hēmerai,,
-korean_names,간규찬,Kan Kyu-ch'an,,Hangul; from Y. Lee,
-korean_names,강감찬,Kang Kam-ch'an,,Hangul; from Y. Lee,
-korean_nonames,내 나름 대로 의 사랑,Nae narŭm taero ŭi sarang,"{""capitalize"": ""first""}",From K-Romanizer,
-korean_nonames,내 마음 속 의 한국 문학,Nae maŭm sok ŭi Han'guk munhak,"{""capitalize"": ""first""}",From K-Romanizer,
+inherited,abcd,ABCD,,
+inherited,ABCD,abcd,"{""dir"": ""r2s""}",
+inherited,ab,90,,
+rot3,abcd,defg,,
+rot3,HIJK,KLMN,,
+rot3,pqrs,Pqrs,"{""capitalize"": ""first""}",
+rot3,pqrs,PQRS,"{""capitalize"": ""all""}",

tests/integration_tests.py

@@ -0,0 +1,58 @@
+from csv import reader
+from difflib import ndiff
+from json import loads as jloads
+from logging import getLogger
+from os import path
+
+from scriptshifter.trans import transliterate
+from tests import TEST_DATA_DIR
+
+logger = getLogger(__name__)
+
+
+def test_sample(dset):
+    """
+    Test an individual sample set and produce a human-readable report.
+
+    Used outside of automated tests.
+
+    @param dset (str): sample set name (without the .csv extension) found in
+    the `data/script_samples` directory.
+    """
+    deltas = []
+    dset_fpath = path.join(TEST_DATA_DIR, "script_samples", dset + ".csv")
+    log_fpath = path.join(TEST_DATA_DIR, f"test_{dset}.log")
+
+    with open(dset_fpath, newline="") as fh:
+        csv = reader(fh)
+        i = 1
+        for row in csv:
+            logger.info(f"CSV row #{i}")
+            i += 1
+            lang, script, rom = row[:3]
+            if not lang:
+                continue
+            opts = jloads(row[3]) if len(row) > 3 and row[3] else {}
+            trans, warnings = transliterate(
+                    script, lang, t_dir="s2r",
+                    capitalize=opts.get("capitalize"), options=opts)
+            if (trans == rom):
+                print(".", end="")
+            else:
+                print("F", end="")
+                deltas.append((lang, script, ndiff([trans], [rom])))
+
+    with open(log_fpath, "w") as fh:
+        # If no deltas, just truncate the file.
+        for lang, script, delta in deltas:
+            fh.write(f"Language: {lang}\n")
+            fh.write(f"Original: {script}\nDiff (result vs. expected):\n")
+            for dline in delta:
+                fh.write(dline.strip() + "\n")
+            fh.write("\n\n")
+
+    ct = len(deltas)
+    if ct > 0:
+        print(f"{ct} failed tests. See report at {log_fpath}")
+    else:
+        print("All tests passed.")

tests/tables/index.yml

@@ -0,0 +1,14 @@
+inherited:
+  name: Test inheritance leaf file
+  marc_code: inh
+  description: Test description.
+cap_base1:
+  name: Test capitalization base 1
+cap_base2:
+  name: Test capitalization base 2
+cap_inherited:
+  name: Test capitalization
+ordering:
+  name: Test ordering
+rot3:
+  name: Test ROT3 hooks

tests/test01_cfg.py

@@ -1,20 +1,22 @@
+from os import environ, unlink
 from unittest import TestCase
 
-from os import environ
+from scriptshifter.tables import get_language
 
-import scriptshifter
 
-from tests import TEST_DATA_DIR, reload_tables
+def setUpModule():
+    from scriptshifter.tables import init_db
+    init_db()
+
+
+def tearDownModule():
+    unlink(environ["TXL_DB_PATH"])
 
 
 class TestConfig(TestCase):
     """ Test configuration parsing. """
-    def setUp(self):
-        environ["TXL_CONFIG_TABLE_DIR"] = TEST_DATA_DIR
-        self.tables = reload_tables()
-
     def test_ordering(self):
-        tbl = self.tables.load_table("ordering")
+        tbl = get_language("ordering")
         exp_order = ["ABCD", "AB", "A", "BCDE", "BCD", "BEFGH", "B"]
 
         self.assertEqual(
@@ -23,19 +25,17 @@ class TestConfig(TestCase):
 
 class TestOverride(TestCase):
     """ Test configuration overrides. """
-    def setUp(self):
-        environ["TXL_CONFIG_TABLE_DIR"] = TEST_DATA_DIR
-        self.tables = reload_tables()
-
     def test_override_map(self):
-        tbl = self.tables.load_table("inherited")
+        tbl = get_language("inherited")
 
-        self.assertEqual(tbl["general"]["name"], "Test inheritance leaf file")
+        self.assertEqual(tbl["label"], "Test inheritance leaf file")
+        self.assertEqual(tbl["marc_code"], "inh")
+        self.assertEqual(tbl["description"], "Test description.")
 
         # Entries are additive.
         self.assertEqual(
                 tbl["roman_to_script"]["ignore"],
-                ["Fritter my wig", "Hi", "Ho", "Thing-um-a-jig"])
+                ("Fritter my wig", "Hi", "Ho", "Thing-um-a-jig"))
         self.assertEqual(
                 tbl["roman_to_script"]["map"],
                 (
@@ -102,34 +102,31 @@ class TestOverride(TestCase):
 
 class TestHooks(TestCase):
     """ Test parsing of hook functions. """
-    def setUp(self):
-        environ["TXL_CONFIG_TABLE_DIR"] = TEST_DATA_DIR
-        self.tables = reload_tables()
-
     def test_rot3(self):
-        tbl = self.tables.load_table("rot3")
+        tbl = get_language("rot3")
 
         self.assertEqual(
-                tbl["script_to_roman"]["hooks"],
-                {
-                    "begin_input_token": [
-                        ("test", scriptshifter.hooks.test.rotate, {"n": -3})
-                    ]
-                })
+            tbl["script_to_roman"]["hooks"],
+            {
+                "begin_input_token": [
+                    {
+                        "module_name": "test",
+                        "fn_name": "rotate",
+                        "kwargs": {"n": -3},
+                    }
+                ]
+            }
+        )
 
 
 class TestDoubleCaps(TestCase):
     """ Test double capitalization configuration. """
-    def setUp(self):
-        environ["TXL_CONFIG_TABLE_DIR"] = TEST_DATA_DIR
-        self.tables = reload_tables()
-
     def test_dcaps_base1(self):
-        cap_base1 = self.tables.load_table("cap_base1")
+        cap_base1 = get_language("cap_base1")
         assert "z︠h︡" in cap_base1["script_to_roman"]["double_cap"]
 
     def test_dcaps_base2(self):
-        cap_base2 = self.tables.load_table("cap_base2")
+        cap_base2 = get_language("cap_base2")
         dcap = cap_base2["script_to_roman"]["double_cap"]
 
         assert len(dcap) == 2
@@ -137,7 +134,7 @@ class TestDoubleCaps(TestCase):
         assert "i︠o︡" in dcap
 
     def test_dcaps_inherited(self):
-        cap_inherited = self.tables.load_table("cap_inherited")
+        cap_inherited = get_language("cap_inherited")
         dcap = cap_inherited["script_to_roman"]["double_cap"]
 
         assert len(dcap) == 1

tests/test02_transliteration.py

@@ -2,18 +2,26 @@ import logging
 
 from unittest import TestCase, TestSuite, TextTestRunner
 from csv import reader
-from glob import glob
 from json import loads as jloads
-from os import environ, path
+from os import environ, path, unlink
 
-from tests import TEST_DATA_DIR, reload_tables
 from scriptshifter.trans import transliterate
 from scriptshifter.tables import get_language
+from tests import TEST_DATA_DIR
 
 
 logger = logging.getLogger(__name__)
 
 
+def setUpModule():
+    from scriptshifter.tables import init_db
+    init_db()
+
+
+def tearDownModule():
+    unlink(environ["TXL_DB_PATH"])
+
+
 class TestTrans(TestCase):
     """
     Test S2R transliteration.
@@ -68,8 +76,6 @@ def make_suite():
     """
     Build parametrized test cases.
     """
-    reload_tables()
-
     suite = TestSuite()
 
     with open(path.join(

tests/test03_capitalization.py

@@ -1,19 +1,22 @@
-from os import environ
+from os import environ, unlink
 from unittest import TestCase
 
 from scriptshifter.trans import transliterate
-from tests import TEST_DATA_DIR, reload_tables
+
+
+def setUpModule():
+    from scriptshifter.tables import init_db
+    init_db()
+
+
+def tearDownModule():
+    unlink(environ["TXL_DB_PATH"])
 
 
 class TestCapitalization(TestCase):
     """
     Test capitalization.
     """
-
-    def setUp(self):
-        environ["TXL_CONFIG_TABLE_DIR"] = TEST_DATA_DIR
-        self.tables = reload_tables()
-
     def test_cap(self):
         tbl = "cap_inherited"
         in_str = "зг іо"

tests/test04_rest_api.py

@@ -1,25 +1,28 @@
 import json
 
-from os import environ
+from os import environ, unlink
 from unittest import TestCase
 
 from scriptshifter.rest_api import app
-from tests import TEST_DATA_DIR, reload_tables
 
 
 EP = "http://localhost:8000"
 
 
+def setUpModule():
+    from scriptshifter.tables import init_db
+    init_db()
+
+
+def tearDownModule():
+    unlink(environ["TXL_DB_PATH"])
+
+
 class TestRestAPI(TestCase):
     """ Test REST API interaction. """
-    def setUp(self):
-        environ["TXL_CONFIG_TABLE_DIR"] = TEST_DATA_DIR
-        # if "TXL_CONFIG_TABLE_DIR" in environ:
-        #     del environ["TXL_CONFIG_TABLE_DIR"]
-        reload_tables()
-
-        # Start webapp.
-        app.testing = True
+    # def setUp(self):
+    #     # Start webapp.
+    #     app.testing = True
 
     def test_health(self):
         with app.test_client() as c:
@@ -35,7 +38,7 @@ class TestRestAPI(TestCase):
 
         data = json.loads(rsp.get_data(as_text=True))
         self.assertIn("inherited", data)
-        self.assertIn("name", data["inherited"])
+        self.assertIn("label", data["inherited"])
         self.assertNotIn("_base1", data)
         self.assertNotIn("_base2", data)
         self.assertNotIn("_base3", data)
@@ -47,14 +50,17 @@ class TestRestAPI(TestCase):
         self.assertEqual(rsp.status_code, 200)
         data = json.loads(rsp.get_data(as_text=True))
 
-        self.assertIn("general", data)
+        self.assertIn("case_sensitive", data)
+        self.assertIn("description", data)
         self.assertIn("roman_to_script", data)
         self.assertIn("map", data["roman_to_script"])
+        self.assertEqual(data["has_r2s"], True)
+        self.assertEqual(data["has_s2r"], False)
         self.assertEqual(data["roman_to_script"]["map"][0], ["ABCD", ""])
 
     def test_trans_api_s2r(self):
         with app.test_client() as c:
-            rsp = c.post("/trans", data={"lang": "rot3", "text": "defg"})
+            rsp = c.post("/trans", json={"lang": "rot3", "text": "defg"})
 
         self.assertEqual(rsp.status_code, 200)
         data = json.loads(rsp.get_data(as_text=True))
@@ -64,7 +70,7 @@ class TestRestAPI(TestCase):
     def test_trans_api_r2s(self):
         with app.test_client() as c:
             rsp = c.post(
-                "/trans", data={
+                "/trans", json={
                     "lang": "rot3",
                     "text": "abcd",
                     "t_dir": "r2s"
@@ -80,7 +86,7 @@ class TestRestAPI(TestCase):
         with app.test_client() as c:
             rsp = c.post(
                 "/trans",
-                data={
+                json={
                     "lang": "rot3",
                     "capitalize": "first",
                     "text": "bcde",