Add security regression tests + pyNEID compat scaffolding

.github/workflows/ci.yml

@@ -0,0 +1,79 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main, develop]
+
+jobs:
+  test:
+    name: test (py${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install package and test deps
+        # `spatial_index` on PyPI has no wheels for Python 3.9+ and no
+        # source distribution, so `pip install .` cannot resolve it on the
+        # modern matrix. Install nexsciTAP with --no-deps and rely on the
+        # test fixture's spatial_index stub (tests/fixtures/stubs/) on
+        # PYTHONPATH for the constants TAP imports. configobj is the only
+        # actually-needed runtime dep and is in requirements-test.txt.
+        # ADQL is installed with --no-deps for the same reason.
+        run: |
+          pip install --upgrade pip
+          pip install -r requirements-test.txt
+          pip install --no-deps "ADQL==1.0.6"
+          pip install --no-deps .
+
+      - name: Lint with ruff
+        run: ruff check .
+
+      - name: Run tests
+        run: pytest tests/ -v
+
+  wheel-platforms:
+    name: wheel build (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Build wheel
+        run: |
+          pip install --upgrade pip build
+          python -m build --wheel
+
+      - name: Verify wheel artifact
+        # See test job for spatial_index PyPI situation. We can't install
+        # the wheel's full deps on modern Python; just confirm the wheel
+        # was built and includes the C extension.
+        run: |
+          ls dist/
+          python -c "
+          import zipfile, glob
+          wheel = glob.glob('dist/*.whl')[0]
+          with zipfile.ZipFile(wheel) as z:
+              names = z.namelist()
+          assert any('writerecs' in n and n.endswith('.so') for n in names), \
+              f'C extension missing from wheel: {names}'
+          print(f'wheel OK: {wheel}')
+          print('contains C extension')
+          "

.gitignore

@@ -1,13 +1,15 @@
 *.[oa]
 *~
+*.so
 /Sphinx/_build
 /TAP/__pycache__
-/TAP/writerecs.cpython-36m-darwin.so
 /build
 /dist
 /final_dist
 /final_wheel
 /nexsciTAP.egg-info
+__pycache__/
+.pytest_cache/
 
 # Workflow artifacts (Claude Code planning/spec docs, not project documentation)
 /docs/superpowers/

TAP/configparam.py

@@ -3,11 +3,12 @@
 #   https://github.com/Caltech-IPAC/nexsciTAP/blob/master/LICENSE
 
 
-import os
 import logging
-import configobj
+import os
 import pprint
 
+import configobj
+
 
 class configParam:
 
@@ -31,7 +32,7 @@ class configParam:
 
     def __init__(self, path, **kwargs):
 
-        pp = pprint.PrettyPrinter(indent=3)
+        pprint.PrettyPrinter(indent=3)
 
 
 
@@ -80,7 +81,7 @@ def __init__(self, path, **kwargs):
             logging.debug('')
             logging.debug('ConfigObj instantiated successfully')
 
-        
+
         ### Web server config parameters ############
 
         self.web = 'WEB'
@@ -120,7 +121,7 @@ def __init__(self, path, **kwargs):
 
         # HTTP_URL  (includingt HTTP_PORT)
 
-        self.httpurl = None 
+        self.httpurl = None
         if ('HTTP_URL' in confobj[self.web]):
             self.httpurl = confobj[self.web]['HTTP_URL']
 
@@ -129,7 +130,7 @@ def __init__(self, path, **kwargs):
             self.msg = 'Failed to find HTTP_URL in config_file'
             raise Exception(self.msg)
 
-        self.port = None 
+        self.port = None
         if('HTTP_PORT' in confobj[self.web]):
             self.port = confobj[self.web]['HTTP_PORT']
 
@@ -139,23 +140,23 @@ def __init__(self, path, **kwargs):
 
         # CGIPGM
 
-        self.cgipgm = None 
+        self.cgipgm = None
         if ('CGI_PGM' in confobj[self.web]):
             self.cgipgm = confobj[self.web]['CGI_PGM']
 
 
         # ARRAYSIZE
 
-        self.arraysize = 10000 
+        self.arraysize = 10000
         if ('ArraySize' in confobj[self.web]):
             self.arraysize = confobj[self.web]['ArraySize']
 
 
         # INFOMSG
 
-        self.infomsg = ''                                                      
-        if('INFOMSG' in confobj[self.web]):                                 
-            self.infomsg = confobj[self.web]['INFOMSG']                     
+        self.infomsg = ''
+        if('INFOMSG' in confobj[self.web]):
+            self.infomsg = confobj[self.web]['INFOMSG']
 
         if self.debug:
             logging.debug('')
@@ -165,10 +166,10 @@ def __init__(self, path, **kwargs):
             logging.debug('arraysize     = %s', self.infomsg)
 
 
-        
+
         ### Configuration ###########################
 
-        # If there was an input configuration 'instance', read 'db_connection' 
+        # If there was an input configuration 'instance', read 'db_connection'
         # and 'sptind_config' section names from there.  Otherwise use the
         # following defaults:
 
@@ -186,7 +187,7 @@ def __init__(self, path, **kwargs):
                 self.status = 'error'
                 self.msg = 'Failed to find DB_CONNECTION in config_file'
                 raise Exception(self.msg)
-           
+
 
             self.sptind_config =  None
             if('SPTIND_CONFIG' in confobj[self.instance]):
@@ -222,9 +223,9 @@ def __init__(self, path, **kwargs):
         self.socket            = None
         self.dbschema          = None
 
-        self.cookiename        = '' 
-        self.accesstbl         = '' 
-        self.usertbl           = '' 
+        self.cookiename        = ''
+        self.accesstbl         = ''
+        self.usertbl           = ''
         self.propfilter        = ''
         self.fileid            = ''
         self.accessid          = ''
@@ -247,7 +248,7 @@ def __init__(self, path, **kwargs):
             if self.debug:
                 logging.debug('')
                 logging.debug('dbms          = %s', self.dbms)
-           
+
 
             # ORACLE Connection
 
@@ -309,15 +310,15 @@ def __init__(self, path, **kwargs):
 
                 # KOA Proprietary Access parameters
 
-                self.cookiename = '' 
+                self.cookiename = ''
                 if('COOKIENAME' in confobj[self.db_connection]):
                     self.cookiename = confobj[self.db_connection]['COOKIENAME']
 
-                self.accesstbl = '' 
+                self.accesstbl = ''
                 if('ACCESS_TBL' in confobj[self.db_connection]):
                     self.accesstbl = confobj[self.db_connection]['ACCESS_TBL']
 
-                self.usertbl = '' 
+                self.usertbl = ''
                 if('USERS_TBL' in confobj[self.db_connection]):
                     self.usertbl = confobj[self.db_connection]['USERS_TBL']
 
@@ -428,7 +429,7 @@ def __init__(self, path, **kwargs):
                     self.msg = 'Failed to MySQL DBMS user ID in config_file'
                     raise Exception(self.msg)
 
- 
+
                 # PASSWORD
 
                 if ('Password' in confobj[self.db_connection]):
@@ -490,7 +491,7 @@ def __init__(self, path, **kwargs):
 
 
                 # DATABASE
-                
+
                 if 'DataBase' in confobj[self.db_connection]:
                     self.database = confobj[self.db_connection]['DataBase']
 
@@ -511,7 +512,7 @@ def __init__(self, path, **kwargs):
                     self.msg = 'Failed to find PostgreSQL username in config_file.'
                     raise Exception(self.msg)
 
- 
+
                 # PASSWORD
 
                 if 'Password' in confobj[self.db_connection]:
@@ -686,5 +687,5 @@ def __init__(self, path, **kwargs):
             logging.debug('      accessid          = ' + str(self.accessid))
             logging.debug('      racol             = ' + str(self.racol))
             logging.debug('      deccol            = ' + str(self.deccol))
-        
-        return 
+
+        return

TAP/datadictionary.py

@@ -3,9 +3,10 @@
 #   https://github.com/Caltech-IPAC/nexsciTAP/blob/master/LICENSE
 
 
-import os
 import logging
+import os
 import re
+
 from astropy.io import ascii
 
 
@@ -92,8 +93,8 @@ def __init__(self, conn, table, connectInfo, **kwargs):
 
         if self.debug:
             logging.debug(f'dbtable = {self.dbtable:s}')
-            logging.debug(f'ddtbl   = ' + str(self.ddtbl))
-            logging.debug(f'ddfile  = ' + str(self.ddfile))
+            logging.debug('ddtbl   = ' + str(self.ddtbl))
+            logging.debug('ddfile  = ' + str(self.ddfile))
 
 
         #
@@ -111,20 +112,20 @@ def __init__(self, conn, table, connectInfo, **kwargs):
             if self.debug:
                 logging.debug('colnames: ' + str(tcolnames))
 
-            i = 0 
+            i = 0
 
             for trow in tdata:
-                    
+
                 dbname = ''
                 desc   = ''
                 if 'name' in tcolnames:
                     dbname = trow['name'].lower()
                     desc   = dbname
-                    
+
                 type = ''
                 if 'intype' in tcolnames:
                     type = trow['intype']
-                    
+
                 unit = ''
                 if 'units' in tcolnames:
                     unit = trow['units']
@@ -147,7 +148,7 @@ def __init__(self, conn, table, connectInfo, **kwargs):
 
                 if len(dbname) > width:
                     width = len(dbname)
-                  
+
 
                 self.colname[i] = dbname
                 i = i+1
@@ -182,7 +183,13 @@ def __init__(self, conn, table, connectInfo, **kwargs):
             else:
                 placeholder = '?'
 
-            sql = "select * from " + self.connectInfo["tap_schema"] + "." + self.connectInfo["columns_table"] + " where lower(table_name) = " \
+            # For SQLite the schema is ATTACHed under a name held in
+            # `tap_schema_file`; `tap_schema` is the file path. For
+            # Oracle/Postgres/MySQL `tap_schema` is the schema name itself.
+            schema_name = self.connectInfo.get("tap_schema_file") \
+                or self.connectInfo["tap_schema"]
+
+            sql = "select * from " + schema_name + "." + self.connectInfo["columns_table"] + " where lower(table_name) = " \
                 + placeholder
 
             if self.debug:
@@ -302,7 +309,7 @@ def __init__(self, conn, table, connectInfo, **kwargs):
             #
             # { while loop
             #
-            
+
             #rows = cursor.fetchmany(self.nfetch)
             rows = cursor.fetchall()
 
@@ -314,7 +321,7 @@ def __init__(self, conn, table, connectInfo, **kwargs):
 
             i = 0
             for row in rows:
-            
+
                 #
                 # { for loop: each row in the file represents
                 #             a column in data dictionary