From ed84dbf7e0474cca959145f2f88f1912a2ec3bfa Mon Sep 17 00:00:00 2001
From: pie-tras <alex.e.pietras@gmail.com>
Date: Tue, 30 May 2023 14:49:41 -0600
Subject: [PATCH 01/12] Add type annotations

---
 src/latis-python-client/latis.py | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/src/latis-python-client/latis.py b/src/latis-python-client/latis.py
index 41ec963..005f20e 100644
--- a/src/latis-python-client/latis.py
+++ b/src/latis-python-client/latis.py
@@ -4,7 +4,7 @@
 import urllib.parse
 
 
-def __datasetWillUseVersion3(baseUrl, dataset, preferVersion2):
+def __datasetWillUseVersion3(baseUrl, dataset, preferVersion2) -> bool:
     if preferVersion2:
         try:
             instanceV2 = LatisInstance(baseUrl, False)
@@ -32,7 +32,7 @@ def __datasetWillUseVersion3(baseUrl, dataset, preferVersion2):
 
 
 def data(baseUrl, dataset, returnType,
-         projections=[], selections=[], operations=[], preferVersion2=False):
+         projections=[], selections=[], operations=[], preferVersion2=False) -> "numpy.ndarray | pd.DataFrame | None":
     latis3 = __datasetWillUseVersion3(baseUrl, dataset, preferVersion2)
     instance = LatisInstance(baseUrl, latis3)
     dsObj = instance.getDataset(dataset, projections, selections, operations)
@@ -46,7 +46,7 @@ def data(baseUrl, dataset, returnType,
 
 
 def download(baseUrl, dataset, filename, fileFormat,
-             projections, selections, operations, preferVersion2=False):
+             projections, selections, operations, preferVersion2=False) -> None:
     latis3 = __datasetWillUseVersion3(preferVersion2)
     instance = LatisInstance(baseUrl, latis3)
     dsObj = instance.getDataset(dataset, projections, selections, operations)
@@ -62,10 +62,10 @@ def __init__(self, baseUrl, latis3):
 
         self.catalog = self.__getCatalog()
 
-    def getDataset(self, name, projections=[], selections=[], operations=[]):
+    def getDataset(self, name, projections=[], selections=[], operations=[]) -> "Dataset":
         return Dataset(self, name, projections, selections, operations)
 
-    def __formatBaseUrl(self):
+    def __formatBaseUrl(self) -> None:
         if not self.baseUrl[-1] == '/':
             self.baseUrl += '/'
         if self.latis3:
@@ -73,7 +73,7 @@ def __formatBaseUrl(self):
         else:
             self.baseUrl += 'dap/'
 
-    def __getCatalog(self):
+    def __getCatalog(self) -> "Catalog":
         return Catalog(self)
 
 
@@ -98,9 +98,9 @@ def __init__(self, latisInstance):
             for i in range(len(self.list)):
                 self.datasets[names[i]] = self.list[i]
 
-    def search(self, searchTerm):
+    def search(self, searchTerm) -> "numpy.ndarray":
         if searchTerm:
-            return [k for k in self.list if searchTerm in k]
+            return numpy.array([k for k in self.list if searchTerm in k])
         else:
             return self.list
 
@@ -119,7 +119,7 @@ def __init__(self, latisInstance, name,
         self.metadata = Metadata(latisInstance, self)
         self.buildQuery()
 
-    def select(self, target="time", start="", end="", inclusive=True):
+    def select(self, target="time", start="", end="", inclusive=True) -> "Dataset":
 
         if start:
             startBound = ">" if inclusive else ">="
@@ -133,16 +133,16 @@ def select(self, target="time", start="", end="", inclusive=True):
 
         return self
 
-    def project(self, projectionList):
+    def project(self, projectionList) -> "Dataset":
         for p in projectionList:
             self.projections.append(p)
         return self
 
-    def operate(self, operation):
+    def operate(self, operation) -> "Dataset":
         self.operations.append(operation)
         return self
 
-    def buildQuery(self):
+    def buildQuery(self) -> str:
         self.query = self.latisInstance.baseUrl + self.name + '.csv?'
 
         self.query += ','.join(urllib.parse.quote(p) for p in self.projections)
@@ -163,7 +163,7 @@ def asNumpy(self):
         self.buildQuery()
         return pd.read_csv(self.query).to_numpy()
 
-    def getFile(self, filename, format='csv'):
+    def getFile(self, filename, format='csv') -> None:
         self.buildQuery()
         suffix = '.' + format
         if '.' not in filename:
@@ -174,13 +174,13 @@ def getFile(self, filename, format='csv'):
             f = open(filename, 'w')
             f.write(csv)
 
-    def clearProjections(self):
+    def clearProjections(self) -> None:
         self.projections = []
 
-    def clearSelections(self):
+    def clearSelections(self) -> None:
         self.selections = []
 
-    def clearOperations(self):
+    def clearOperations(self) -> None:
         self.operations = []
 
 

From 947f5a1d7dc56c32c6b1938c7ea7595aec6e1a0a Mon Sep 17 00:00:00 2001
From: pie-tras <alex.e.pietras@gmail.com>
Date: Mon, 5 Jun 2023 17:54:22 -0600
Subject: [PATCH 02/12] Add missing type annotations

---
 src/latis-python-client/latis.py | 54 +++++++++++++++++++-------------
 1 file changed, 33 insertions(+), 21 deletions(-)

diff --git a/src/latis-python-client/latis.py b/src/latis-python-client/latis.py
index 8b79adc..0174743 100644
--- a/src/latis-python-client/latis.py
+++ b/src/latis-python-client/latis.py
@@ -9,9 +9,10 @@
 import pandas as pd
 import requests
 import urllib.parse
+from typing import Dict, List, Union, Optional
 
 
-def _datasetWillUseVersion3(baseUrl, dataset, preferVersion2) -> bool:
+def _datasetWillUseVersion3(baseUrl: str, dataset: str, preferVersion2: bool) -> bool:
     """Checks which version of LaTiS dataset will be used.
 
     Args:
@@ -49,8 +50,11 @@ def _datasetWillUseVersion3(baseUrl, dataset, preferVersion2) -> bool:
             return False
 
 
-def data(baseUrl, dataset, returnType,
-         projections=[], selections=[], operations=[], preferVersion2=False) -> "numpy.ndarray | pd.DataFrame | None":
+def data(baseUrl: str, dataset: str, returnType: str,
+         projections: Optional[List[str]] = [],
+         selections: Optional[List[str]] = [],
+         operations: Optional[List[str]] = [],
+         preferVersion2: bool = False) -> Union[numpy.ndarray, pd.DataFrame, None]:
     """
     Retrieves a dataset from LaTiS instance.
 
@@ -82,9 +86,11 @@ def data(baseUrl, dataset, returnType,
         return None
 
 
-def download(baseUrl, dataset, filename, fileFormat,
-             projections=[], selections=[], operations=[],
-             preferVersion2=False) -> None:
+def download(baseUrl: str, dataset: str, filename: str, fileFormat: str,
+             projections: Optional[List[str]] = [],
+             selections: Optional[List[str]] = [],
+             operations: Optional[List[str]] = [],
+             preferVersion2: bool = False) -> None:
     """Downloads a dataset from LaTiS instance to file.
 
     Args:
@@ -123,14 +129,17 @@ class LatisInstance:
             False otherwise.
     """
 
-    def __init__(self, baseUrl, latis3):
+    def __init__(self, baseUrl: str, latis3: bool):
         self.baseUrl = baseUrl
         self.latis3 = latis3
         self._formatBaseUrl()
 
         self.catalog = self._getCatalog()
 
-    def getDataset(self, name, projections=[], selections=[], operations=[]) -> "Dataset":
+    def getDataset(self, name: str,
+                   projections: Optional[List[str]] = [],
+                   selections: Optional[List[str]] = [],
+                   operations: Optional[List[str]] = []) -> "Dataset":
         """
         Creates and returns a Dataset object.
 
@@ -146,7 +155,7 @@ def getDataset(self, name, projections=[], selections=[], operations=[]) -> "Dat
         """
         return Dataset(self, name, projections, selections, operations)
 
-    def _formatBaseUrl(self):
+    def _formatBaseUrl(self) -> None:
         """
         Appends dap2 or dap3 depending on LaTiS version.
         """
@@ -170,7 +179,7 @@ class Catalog:
         list (numpy.ndarray): List of all dataset LaTiS names in catalog.
     """
 
-    def __init__(self, latisInstance):
+    def __init__(self, latisInstance: LatisInstance):
         """Initializes the Catalog Object.
 
         Populates a dictionary and list of avaiable datasets.
@@ -197,7 +206,7 @@ def __init__(self, latisInstance):
             for i in range(len(self.list)):
                 self.datasets[names[i]] = self.list[i]
 
-    def search(self, searchTerm) -> "numpy.ndarray":
+    def search(self, searchTerm: str) -> "numpy.ndarray":
         """
         Filters the catalog by a search term.
 
@@ -230,8 +239,10 @@ class Dataset:
         metadata (Metadata): Metadata object.
     """
 
-    def __init__(self, latisInstance, name,
-                 projections=[], selections=[], operations=[]):
+    def __init__(self, latisInstance: LatisInstance, name: str,
+                 projections: Optional[List[str]] = [],
+                 selections: Optional[List[str]] = [],
+                 operations: Optional[List[str]] = []):
         """Initializes the Dataset object and Metadata object.
 
         Initilizes lists of projections, selections and operations.
@@ -257,7 +268,8 @@ def __init__(self, latisInstance, name,
         self.metadata = Metadata(latisInstance, self)
         self.buildQuery()
 
-    def select(self, target="time", start="", end="", inclusiveStart=True, inclusiveEnd=False) -> "Dataset":
+    def select(self, target: str = "time", start: str = "", end: str = "",
+               inclusiveStart: bool = True, inclusiveEnd: bool = False) -> "Dataset":
         """
         Defines a selection parameters for the dataset query.
 
@@ -287,12 +299,12 @@ def select(self, target="time", start="", end="", inclusiveStart=True, inclusive
 
         return self
 
-    def project(self, projectionList) -> "Dataset":
+    def project(self, projectionList: list[str]) -> "Dataset":
         """
         Appends a projection list for the dataset query.
 
         Args:
-            projectionList (List[str]): List of projections.
+            projectionList (list[str]): List of projections.
 
         Returns:
             Dataset: The Dataset instance, allowing for method chaining.
@@ -302,7 +314,7 @@ def project(self, projectionList) -> "Dataset":
             self.projections.append(p)
         return self
 
-    def operate(self, operation) -> "Dataset":
+    def operate(self, operation: str) -> "Dataset":
         """
         Append an operation for the dataset query.
 
@@ -337,7 +349,7 @@ def buildQuery(self) -> str:
 
         return self.query
 
-    def asPandas(self):
+    def asPandas(self) -> "pd.DataFrame":
         """
         Returns data from LaTiS API as pandas dataframe.
 
@@ -348,7 +360,7 @@ def asPandas(self):
         self.buildQuery()
         return pd.read_csv(self.query)
 
-    def asNumpy(self):
+    def asNumpy(self) -> "numpy.DataFrame":
         """
         Returns data from LaTiS API as numpy array.
 
@@ -359,7 +371,7 @@ def asNumpy(self):
         self.buildQuery()
         return pd.read_csv(self.query).to_numpy()
 
-    def getFile(self, filename, format='csv') -> None:
+    def getFile(self, filename: str, format: str = 'csv') -> None:
         """
         Writes the dataset data to a local file.
 
@@ -402,7 +414,7 @@ class Metadata:
             variable metadata.
     """
 
-    def __init__(self, latisInstance, dataset):
+    def __init__(self, latisInstance: LatisInstance, dataset: Dataset):
         """Initializes the Metadata object; retrieves the metadata properties.
 
         Retrieves metadata from the provided LaTiS instance and dataset. The

From 2eade0a39b8b78daf97a670d6e25b3c5650bd880 Mon Sep 17 00:00:00 2001
From: pie-tras <alex.e.pietras@gmail.com>
Date: Tue, 6 Jun 2023 16:14:34 -0600
Subject: [PATCH 03/12] rename module

---
 src/{latis-python-client => latispy}/__init__.py | 0
 src/{latis-python-client => latispy}/latis.py    | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename src/{latis-python-client => latispy}/__init__.py (100%)
 rename src/{latis-python-client => latispy}/latis.py (100%)

diff --git a/src/latis-python-client/__init__.py b/src/latispy/__init__.py
similarity index 100%
rename from src/latis-python-client/__init__.py
rename to src/latispy/__init__.py
diff --git a/src/latis-python-client/latis.py b/src/latispy/latis.py
similarity index 100%
rename from src/latis-python-client/latis.py
rename to src/latispy/latis.py

From d8c2bc2e61eb53402e9a103c23c452ed3fdc335f Mon Sep 17 00:00:00 2001
From: pie-tras <alex.e.pietras@gmail.com>
Date: Wed, 7 Jun 2023 14:32:21 -0600
Subject: [PATCH 04/12] Resolve mypy errors in src dir

---
 requirements.txt     |  1 +
 src/latispy/latis.py | 73 ++++++++++++++++++++++----------------------
 tests/example.py     |  4 +--
 tests/test.py        |  4 +--
 4 files changed, 42 insertions(+), 40 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 40a7852..938cb35 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+mypy==3.10
 flake8
 pandas
 requests
diff --git a/src/latispy/latis.py b/src/latispy/latis.py
index 0174743..9818a25 100644
--- a/src/latispy/latis.py
+++ b/src/latispy/latis.py
@@ -5,11 +5,11 @@
 or pandas format, and optionally download the dataset to a file.
 """
 
-import numpy
+import numpy as np
 import pandas as pd
 import requests
 import urllib.parse
-from typing import Dict, List, Union, Optional
+from typing import List, Dict, Any, Union, Optional
 
 
 def _datasetWillUseVersion3(baseUrl: str, dataset: str, preferVersion2: bool) -> bool:
@@ -51,10 +51,10 @@ def _datasetWillUseVersion3(baseUrl: str, dataset: str, preferVersion2: bool) ->
 
 
 def data(baseUrl: str, dataset: str, returnType: str,
-         projections: Optional[List[str]] = [],
-         selections: Optional[List[str]] = [],
-         operations: Optional[List[str]] = [],
-         preferVersion2: bool = False) -> Union[numpy.ndarray, pd.DataFrame, None]:
+         projections: Optional[List[str]] = None,
+         selections: Optional[List[str]] = None,
+         operations: Optional[List[str]] = None,
+         preferVersion2: bool = False) -> Union[np.ndarray, pd.DataFrame, None]:
     """
     Retrieves a dataset from LaTiS instance.
 
@@ -71,7 +71,7 @@ def data(baseUrl: str, dataset: str, returnType: str,
         preferVersion2 (bool, optional): If True, prefer LaTiS version 2.
 
     Returns:
-        numpy.ndarray or pandas.DataFrame or None: Dataset data.
+        np.ndarray or pandas.DataFrame or None: Dataset data.
     """
 
     latis3 = _datasetWillUseVersion3(baseUrl, dataset, preferVersion2)
@@ -87,9 +87,9 @@ def data(baseUrl: str, dataset: str, returnType: str,
 
 
 def download(baseUrl: str, dataset: str, filename: str, fileFormat: str,
-             projections: Optional[List[str]] = [],
-             selections: Optional[List[str]] = [],
-             operations: Optional[List[str]] = [],
+             projections: Optional[List[str]] = None,
+             selections: Optional[List[str]] = None,
+             operations: Optional[List[str]] = None,
              preferVersion2: bool = False) -> None:
     """Downloads a dataset from LaTiS instance to file.
 
@@ -130,16 +130,16 @@ class LatisInstance:
     """
 
     def __init__(self, baseUrl: str, latis3: bool):
-        self.baseUrl = baseUrl
-        self.latis3 = latis3
+        self.baseUrl: str = baseUrl
+        self.latis3: bool = latis3
         self._formatBaseUrl()
 
-        self.catalog = self._getCatalog()
+        self.catalog: Catalog = self._getCatalog()
 
     def getDataset(self, name: str,
-                   projections: Optional[List[str]] = [],
-                   selections: Optional[List[str]] = [],
-                   operations: Optional[List[str]] = []) -> "Dataset":
+                   projections: Optional[List[str]] = None,
+                   selections: Optional[List[str]] = None,
+                   operations: Optional[List[str]] = None) -> "Dataset":
         """
         Creates and returns a Dataset object.
 
@@ -176,7 +176,7 @@ class Catalog:
     Attributes:
         datasets (dict of {str: str}): A dictionary mapping formatted
             dataset names to their LaTiS names.
-        list (numpy.ndarray): List of all dataset LaTiS names in catalog.
+        list (np.ndarray): List of all dataset LaTiS names in catalog.
     """
 
     def __init__(self, latisInstance: LatisInstance):
@@ -189,13 +189,14 @@ def __init__(self, latisInstance: LatisInstance):
                 a catalog from.
         """
 
-        self.datasets = {}
+        self.datasets: dict = {}
+        self.list: np.ndarray[str, np.dtype[Any]]
 
         if latisInstance.latis3:
             js = requests.get(latisInstance.baseUrl).json()
             dataset = js['dataset']
-            titles = numpy.array([k['title'] for k in dataset])
-            self.list = numpy.array([k['identifier'] for k in dataset])
+            titles = np.array([k['title'] for k in dataset])
+            self.list = np.array([k['identifier'] for k in dataset])
             for i in range(len(self.list)):
                 self.datasets[titles[i]] = self.list[i]
         else:
@@ -206,7 +207,7 @@ def __init__(self, latisInstance: LatisInstance):
             for i in range(len(self.list)):
                 self.datasets[names[i]] = self.list[i]
 
-    def search(self, searchTerm: str) -> "numpy.ndarray":
+    def search(self, searchTerm: str) -> "np.ndarray":
         """
         Filters the catalog by a search term.
 
@@ -221,7 +222,7 @@ def search(self, searchTerm: str) -> "numpy.ndarray":
         """
 
         if searchTerm:
-            return numpy.array([k for k in self.list if searchTerm in k])
+            return np.array([k for k in self.list if searchTerm in k])
         else:
             return self.list
 
@@ -240,9 +241,9 @@ class Dataset:
     """
 
     def __init__(self, latisInstance: LatisInstance, name: str,
-                 projections: Optional[List[str]] = [],
-                 selections: Optional[List[str]] = [],
-                 operations: Optional[List[str]] = []):
+                 projections: Optional[List[str]] = None,
+                 selections: Optional[List[str]] = None,
+                 operations: Optional[List[str]] = None):
         """Initializes the Dataset object and Metadata object.
 
         Initilizes lists of projections, selections and operations.
@@ -258,14 +259,14 @@ def __init__(self, latisInstance: LatisInstance, name: str,
             operations (list[str], optional): List of operations to apply.
         """
 
-        self.latisInstance = latisInstance
-        self.name = name
-        self.projections = list(projections)
-        self.selections = list(selections)
-        self.operations = list(operations)
-        self.query = None
+        self.latisInstance: LatisInstance = latisInstance
+        self.name: str = name
+        self.projections: list[str] = [] if projections is None else projections
+        self.selections: list[str] = [] if selections is None else selections
+        self.operations: list[str] = [] if operations is None else operations
+        self.query: str = ""
 
-        self.metadata = Metadata(latisInstance, self)
+        self.metadata: Metadata = Metadata(latisInstance, self)
         self.buildQuery()
 
     def select(self, target: str = "time", start: str = "", end: str = "",
@@ -349,7 +350,7 @@ def buildQuery(self) -> str:
 
         return self.query
 
-    def asPandas(self) -> "pd.DataFrame":
+    def asPandas(self) -> Union[pd.DataFrame, None]:
         """
         Returns data from LaTiS API as pandas dataframe.
 
@@ -360,12 +361,12 @@ def asPandas(self) -> "pd.DataFrame":
         self.buildQuery()
         return pd.read_csv(self.query)
 
-    def asNumpy(self) -> "numpy.DataFrame":
+    def asNumpy(self) -> Union[np.ndarray, None]:
         """
         Returns data from LaTiS API as numpy array.
 
         Returns:
-            numpy.ndarray: Data as numpy array.
+            np.ndarray: Data as numpy array.
         """
 
         self.buildQuery()
@@ -426,7 +427,7 @@ def __init__(self, latisInstance: LatisInstance, dataset: Dataset):
             dataset (Dataset): The dataset for which to retrieve metadata.
         """
 
-        self.properties = {}
+        self.properties: dict = {}
 
         if latisInstance.latis3:
             q = latisInstance.baseUrl + dataset.name + '.meta'
diff --git a/tests/example.py b/tests/example.py
index a6a0ee5..882e22a 100644
--- a/tests/example.py
+++ b/tests/example.py
@@ -6,10 +6,10 @@
 
 if platform.system() == 'Windows':
     libPath = libPath.replace('\\tests',
-                              '\\src\\latis-python-client')
+                              '\\src\\latispy')
 else:
     libPath = libPath.replace('/tests',
-                              '/src/latis-python-client')
+                              '/src/latispy')
 
 sys.path.insert(0, libPath)
 
diff --git a/tests/test.py b/tests/test.py
index faf3ea4..5b626df 100644
--- a/tests/test.py
+++ b/tests/test.py
@@ -7,10 +7,10 @@
 
 if platform.system() == 'Windows':
     libPath = libPath.replace('\\tests',
-                              '\\src\\latis-python-client')
+                              '\\src\\latispy')
 else:
     libPath = libPath.replace('/tests',
-                              '/src/latis-python-client')
+                              '/src/latispy')
 
 sys.path.insert(0, libPath)
 

From 05c148afc599df8205cfa337cc23c85776796c2d Mon Sep 17 00:00:00 2001
From: pie-tras <alex.e.pietras@gmail.com>
Date: Wed, 7 Jun 2023 14:37:30 -0600
Subject: [PATCH 05/12] finish resolving mypy errors in all dirs

---
 src/latispy/{latis.py => latispy.py} |  0
 tests/example.py                     |  2 +-
 tests/test.py                        | 84 ----------------------------
 3 files changed, 1 insertion(+), 85 deletions(-)
 rename src/latispy/{latis.py => latispy.py} (100%)
 delete mode 100644 tests/test.py

diff --git a/src/latispy/latis.py b/src/latispy/latispy.py
similarity index 100%
rename from src/latispy/latis.py
rename to src/latispy/latispy.py
diff --git a/tests/example.py b/tests/example.py
index 882e22a..e36cb7e 100644
--- a/tests/example.py
+++ b/tests/example.py
@@ -13,7 +13,7 @@
 
 sys.path.insert(0, libPath)
 
-import latis
+import latispy as latis
 
 
 def testShortcuts():
diff --git a/tests/test.py b/tests/test.py
deleted file mode 100644
index 5b626df..0000000
--- a/tests/test.py
+++ /dev/null
@@ -1,84 +0,0 @@
-import os
-import sys
-import platform
-import random
-
-libPath = os.path.dirname(os.path.realpath(__file__))
-
-if platform.system() == 'Windows':
-    libPath = libPath.replace('\\tests',
-                              '\\src\\latispy')
-else:
-    libPath = libPath.replace('/tests',
-                              '/src/latispy')
-
-sys.path.insert(0, libPath)
-
-import latis
-
-
-def testRandomBaseUrl(baseUrl, latis3, seed, maxDatasets=None):
-    random.seed(seed)
-
-    instance = latis.LatisInstance(
-        baseUrl=baseUrl,
-        latis3=latis3)
-
-    datasets = list(instance.catalog.datasets.values())
-
-    if maxDatasets:
-        index = 0
-        picks = [0]
-        dataset_picks = [datasets[0]]
-        for i in range(maxDatasets):
-            while index in picks:
-                index = random.randrange(0, len(datasets) - 1)
-            picks.append(index)
-            dataset_picks.append(datasets[index])
-
-        datasets = dataset_picks
-
-    for d in datasets:
-        try:
-            dsObj = instance.getDataset(d)
-            testDataset(dsObj)
-        except Exception:
-            print("Failed to test or get a dataset.")
-            pass
-
-
-def testDataset(dsObj):
-    metadata = dsObj.metadata.properties
-
-    projections = []
-
-    metadata_keys = list(metadata.keys())
-
-    if metadata_keys:
-
-        if metadata_keys[0] == 'time':
-
-            projections.append(metadata_keys[0])
-            if len(metadata_keys) == 2:
-                projections.append(metadata_keys[1])
-            else:
-                projections.append(metadata_keys[random.randrange(1, len(metadata_keys) - 1)])
-
-            # dsObj.operate('formatTime(yyyy.MM.dd)').project(projections).select(metadata_keys[0], start='0')
-
-            dsObj.operate('formatTime(yyyy.MM.dd)').select(metadata_keys[0], start='0').project(projections)
-
-            print(dsObj.name)
-            print(dsObj.projections, dsObj.selections, dsObj.operations)
-            print(dsObj.buildQuery())
-
-            numpyData = dsObj.asNumpy()
-            print(numpyData)
-
-
-testRandomBaseUrl('https://lasp.colorado.edu/lisird/latis',
-                  False, 23423, maxDatasets=4)
-
-
-testRandomBaseUrl('https://lasp.colorado.edu/lisird/latis',
-                  True, 23423, maxDatasets=4)

From ed7cda010c96a47e0427e104cda3be2eb150106d Mon Sep 17 00:00:00 2001
From: pie-tras <alex.e.pietras@gmail.com>
Date: Wed, 7 Jun 2023 14:52:15 -0600
Subject: [PATCH 06/12] Fix issue with example.py

---
 tests/example.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/example.py b/tests/example.py
index e36cb7e..3ac64bc 100644
--- a/tests/example.py
+++ b/tests/example.py
@@ -80,8 +80,8 @@ def testCore():
 
     print('\nCreating Queries\n')
     # 4 - Create queries
-    clsRadioFluxF15.select('time<0')
-    sorceMGIndex.select('time<2452705')
+    clsRadioFluxF15.select(start='0')
+    sorceMGIndex.select(start='2452705')
     clsRadioFluxF107.project(['time', 'absolute_f107']).select(start='1953', end='1954').select(target='absolute_f107', end='70').operate('formatTime(yyyy.MM.dd)')
     # clsRadioFluxF107.project(['time','absolute_f107']).operate('formatTime(yyyy.MM.dd)').select(target='absolute_f107', end='70').select(start='1953', end='1954')
 

From 3243795715816b897f28f088a9983b582d52195b Mon Sep 17 00:00:00 2001
From: pie-tras <alex.e.pietras@gmail.com>
Date: Wed, 7 Jun 2023 15:45:22 -0600
Subject: [PATCH 07/12] Update module naming

---
 src/{latispy => latis}/__init__.py          | 0
 src/{latispy/latispy.py => latis/client.py} | 0
 tests/example.py                            | 6 +++---
 3 files changed, 3 insertions(+), 3 deletions(-)
 rename src/{latispy => latis}/__init__.py (100%)
 rename src/{latispy/latispy.py => latis/client.py} (100%)

diff --git a/src/latispy/__init__.py b/src/latis/__init__.py
similarity index 100%
rename from src/latispy/__init__.py
rename to src/latis/__init__.py
diff --git a/src/latispy/latispy.py b/src/latis/client.py
similarity index 100%
rename from src/latispy/latispy.py
rename to src/latis/client.py
diff --git a/tests/example.py b/tests/example.py
index 3ac64bc..6843573 100644
--- a/tests/example.py
+++ b/tests/example.py
@@ -6,14 +6,14 @@
 
 if platform.system() == 'Windows':
     libPath = libPath.replace('\\tests',
-                              '\\src\\latispy')
+                              '\\src\\')
 else:
     libPath = libPath.replace('/tests',
-                              '/src/latispy')
+                              '/src/')
 
 sys.path.insert(0, libPath)
 
-import latispy as latis
+import latis.client as latis
 
 
 def testShortcuts():

From ddae01a9174a6f4a4475fd75406b1d91348f98ee Mon Sep 17 00:00:00 2001
From: pie-tras <alex.e.pietras@gmail.com>
Date: Mon, 12 Jun 2023 19:35:20 -0600
Subject: [PATCH 08/12] query checking

---
 src/latis/client.py | 65 +++++++++++++++++++++++++------------
 tests/example.py    | 78 +++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 121 insertions(+), 22 deletions(-)

diff --git a/src/latis/client.py b/src/latis/client.py
index 9818a25..641aadb 100644
--- a/src/latis/client.py
+++ b/src/latis/client.py
@@ -49,6 +49,16 @@ def _datasetWillUseVersion3(baseUrl: str, dataset: str, preferVersion2: bool) ->
 
             return False
 
+def _checkQuery(query, expectTextError=True):
+    r = requests.get(query)
+    if r.status_code > 399:
+        if expectTextError:
+            print(r.text)
+        else:
+            print(r.status_code)
+        return False
+    else:
+        return True
 
 def data(baseUrl: str, dataset: str, returnType: str,
          projections: Optional[List[str]] = None,
@@ -193,19 +203,25 @@ def __init__(self, latisInstance: LatisInstance):
         self.list: np.ndarray[str, np.dtype[Any]]
 
         if latisInstance.latis3:
-            js = requests.get(latisInstance.baseUrl).json()
-            dataset = js['dataset']
-            titles = np.array([k['title'] for k in dataset])
-            self.list = np.array([k['identifier'] for k in dataset])
-            for i in range(len(self.list)):
-                self.datasets[titles[i]] = self.list[i]
+            if _checkQuery(latisInstance.baseUrl, expectTextError=False):
+                js = requests.get(latisInstance.baseUrl).json()
+                dataset = js['dataset']
+                titles = np.array([k['title'] for k in dataset])
+                self.list = np.array([k['identifier'] for k in dataset])
+                for i in range(len(self.list)):
+                    self.datasets[titles[i]] = self.list[i]
+            else:
+                self.list = np.array([])
         else:
             q = latisInstance.baseUrl + 'catalog.csv'
-            df = pd.read_csv(q)
-            names = df['name']
-            self.list = df['accessURL'].to_numpy()
-            for i in range(len(self.list)):
-                self.datasets[names[i]] = self.list[i]
+            if _checkQuery(q):
+                df = pd.read_csv(q)
+                names = df['name']
+                self.list = df['accessURL'].to_numpy()
+                for i in range(len(self.list)):
+                    self.datasets[names[i]] = self.list[i]
+            else:
+                self.list = np.array([])
 
     def search(self, searchTerm: str) -> "np.ndarray":
         """
@@ -348,7 +364,10 @@ def buildQuery(self) -> str:
         for o in self.operations:
             self.query = self.query + '&' + urllib.parse.quote(o)
 
-        return self.query
+        if _checkQuery(self.query):
+            return self.query
+        else:
+            return None
 
     def asPandas(self) -> Union[pd.DataFrame, None]:
         """
@@ -358,8 +377,10 @@ def asPandas(self) -> Union[pd.DataFrame, None]:
             pandas.DataFrame: Data as pandas dataframe.
         """
 
-        self.buildQuery()
-        return pd.read_csv(self.query)
+        if self.buildQuery():
+            return pd.read_csv(self.query)
+        else:
+            return None
 
     def asNumpy(self) -> Union[np.ndarray, None]:
         """
@@ -369,8 +390,10 @@ def asNumpy(self) -> Union[np.ndarray, None]:
             np.ndarray: Data as numpy array.
         """
 
-        self.buildQuery()
-        return pd.read_csv(self.query).to_numpy()
+        if self.buildQuery():
+            return pd.read_csv(self.query).to_numpy()
+        else:
+            return None
 
     def getFile(self, filename: str, format: str = 'csv') -> None:
         """
@@ -431,9 +454,11 @@ def __init__(self, latisInstance: LatisInstance, dataset: Dataset):
 
         if latisInstance.latis3:
             q = latisInstance.baseUrl + dataset.name + '.meta'
-            variables = pd.read_json(q)['variable']
-            for i in range(len(variables)):
-                self.properties[variables[i]['id']] = variables[i]
+            if _checkQuery(q):
+                variables = pd.read_json(q)['variable']
+                for i in range(len(variables)):
+                    self.properties[variables[i]['id']] = variables[i]
         else:
             q = latisInstance.baseUrl + dataset.name + '.jsond?first()'
-            self.properties = pd.read_json(q).iloc[1][0]
+            if _checkQuery(q):
+                self.properties = pd.read_json(q).iloc[1][0]
diff --git a/tests/example.py b/tests/example.py
index 6843573..7fe97ea 100644
--- a/tests/example.py
+++ b/tests/example.py
@@ -119,6 +119,80 @@ def testCore():
     clsRadioFluxF15.getFile('cls_radio_flux_f15', 'txt')
     clsRadioFluxF15.getFile('cls_radio_flux_f15.data')
 
+def testErrors():
+    testLatis2Np = latis.data(
+        'https://lasp.colorado.edu/lisird/latis',
+        'cls_radio_flux_f83', 'NUMPY', operations=['time<0'],
+        preferVersion2=False)
+
+    print(testLatis2Np)
+
+    instance = latis.LatisInstance(
+        baseUrl='https://lasp.colorado.edu/lisird/latis',
+        latis3=False)
+    
+    instance3 = latis.LatisInstance(
+        baseUrl='https://lasp.colorado.edu/lisird/latis',
+        latis3=True)   
+
+    instancebad = latis.LatisInstance(
+        baseUrl='https://lasp.colorado.edu/lis3rd/latis',
+        latis3=True)
+
+    print('\nSearch Catalog\n')
+    # 2 - Search Catalog
+    # print(instance.catalog.list) # (Full catalog)
+    print(instancebad.catalog.datasets)
+    print(instancebad.catalog.search('so3rce'))
+
+    print('\nGetting Datasets\n')
+    # 3 - Get dataset objects
+    clsRadioFluxF8 = instance.getDataset('cls_radio3_flux_f8')
+    clsRadioFluxF15 = instance.getDataset('cls_radi3o_flux_f15')
+    sorceMGIndex = instance3.getDataset('sorce_mg3_index')
+    clsRadioFluxF107 = instance.getDataset('cls_radio3_flux_absolute_f107')
+
+    print('\nCreating Queries\n')
+    # 4 - Create queries
+    clsRadioFluxF15.select(start='0')
+    sorceMGIndex.select(start='-19023')
+    clsRadioFluxF107.project(['232', '23231']).select(start='A', end='QERWEEWD').select(target='absolute_f107', end='70').operate('formatTime(yyyy.MM.dd)')
+    # clsRadioFluxF107.project(['time','absolute_f107']).operate('formatTime(yyyy.MM.dd)').select(target='absolute_f107', end='70').select(start='1953', end='1954')
+
+    print(clsRadioFluxF8.buildQuery())
+    print(clsRadioFluxF15.buildQuery())
+    print(clsRadioFluxF107.buildQuery())
+
+    print('\nGet Metadata\n')
+    # 5 - Get metadata
+    print(clsRadioFluxF15.metadata.properties)
+    print(clsRadioFluxF8.metadata.properties)
+    print(sorceMGIndex.metadata.properties)
+
+    print('\nGet data\n')
+    # 6 - Get data
+
+    print("clsRadioFluxF15:", clsRadioFluxF15.projections,
+          clsRadioFluxF15.selections, clsRadioFluxF15.operations)
+    print("clsRadioFluxF8:", clsRadioFluxF8.projections,
+          clsRadioFluxF8.selections, clsRadioFluxF8.operations)
+    print("clsRadioFluxF107:", clsRadioFluxF107.projections,
+          clsRadioFluxF107.selections, clsRadioFluxF107.operations)
+    print("sorceMGIndex:", sorceMGIndex.projections,
+          sorceMGIndex.selections, sorceMGIndex.operations)
+
+    pandasDF = clsRadioFluxF15.asPandas()
+    print(pandasDF)
+    numpy = clsRadioFluxF107.asNumpy()
+    print(numpy)
+    mgData = sorceMGIndex.asPandas()
+    print(mgData)
+
+    # 7 - Get file
+    clsRadioFluxF15.getFile('cls_radio_flux_f15')
+    clsRadioFluxF15.getFile('cls_radio_flux_f15', 'txt')
+    clsRadioFluxF15.getFile('cls_radio_flux_f15.data')
 
-testShortcuts()
-testCore()
+testErrors()
+# testShortcuts()
+# testCore()

From 67f5cb805b178e0f44828849d9cabcec912e820b Mon Sep 17 00:00:00 2001
From: pie-tras <alex.e.pietras@gmail.com>
Date: Wed, 14 Jun 2023 17:16:05 -0600
Subject: [PATCH 09/12] better logging

---
 src/latis/client.py | 51 +++++++++++++++++++++++++++++++--------------
 tests/example.py    | 45 +++------------------------------------
 2 files changed, 38 insertions(+), 58 deletions(-)

diff --git a/src/latis/client.py b/src/latis/client.py
index 641aadb..471dc43 100644
--- a/src/latis/client.py
+++ b/src/latis/client.py
@@ -5,10 +5,12 @@
 or pandas format, and optionally download the dataset to a file.
 """
 
+import logging
 import numpy as np
 import pandas as pd
 import requests
 import urllib.parse
+
 from typing import List, Dict, Any, Union, Optional
 
 
@@ -31,7 +33,7 @@ def _datasetWillUseVersion3(baseUrl: str, dataset: str, preferVersion2: bool) ->
 
             return False
         except Exception:
-            print("[WARN]: " + dataset +
+            logging.warning(dataset +
                   " cannot be accessed through LaTiS version 2." +
                   " Auto switching to version 3.")
 
@@ -43,7 +45,7 @@ def _datasetWillUseVersion3(baseUrl: str, dataset: str, preferVersion2: bool) ->
 
             return True
         except Exception:
-            print("[WARN]: " + dataset +
+            logging.warning(dataset +
                   " cannot be accessed through LaTiS version 3." +
                   " Auto switching to version 2.")
 
@@ -53,9 +55,9 @@ def _checkQuery(query, expectTextError=True):
     r = requests.get(query)
     if r.status_code > 399:
         if expectTextError:
-            print(r.text)
+            logging.error("Query is invalid: " + query + " returned: " + r.text)
         else:
-            print(r.status_code)
+            logging.error("Query is invalid: " + query + " returned: " + str(r.status_code))
         return False
     else:
         return True
@@ -88,6 +90,8 @@ def data(baseUrl: str, dataset: str, returnType: str,
     instance = LatisInstance(baseUrl, latis3)
     dsObj = instance.getDataset(dataset, projections, selections, operations)
 
+    returnType = returnType.upper()
+
     if returnType == 'NUMPY':
         return dsObj.asNumpy()
     elif returnType == 'PANDAS':
@@ -211,6 +215,7 @@ def __init__(self, latisInstance: LatisInstance):
                 for i in range(len(self.list)):
                     self.datasets[titles[i]] = self.list[i]
             else:
+                logging.error("Cannot populate catalog. Query was None.")
                 self.list = np.array([])
         else:
             q = latisInstance.baseUrl + 'catalog.csv'
@@ -221,6 +226,7 @@ def __init__(self, latisInstance: LatisInstance):
                 for i in range(len(self.list)):
                     self.datasets[names[i]] = self.list[i]
             else:
+                logging.error("Cannot populate catalog. Query was None.")
                 self.list = np.array([])
 
     def search(self, searchTerm: str) -> "np.ndarray":
@@ -364,10 +370,10 @@ def buildQuery(self) -> str:
         for o in self.operations:
             self.query = self.query + '&' + urllib.parse.quote(o)
 
-        if _checkQuery(self.query):
-            return self.query
-        else:
-            return None
+        if not _checkQuery(self.query):
+            self.query = None
+        
+        return self.query
 
     def asPandas(self) -> Union[pd.DataFrame, None]:
         """
@@ -406,15 +412,24 @@ def getFile(self, filename: str, format: str = 'csv') -> None:
             format (str): Format of file. Defaults to 'csv'.
         """
 
-        self.buildQuery()
-        suffix = '.' + format
-        if '.' not in filename:
-            filename += suffix
+        
+        validFormats = ['asc', 'bin', 'csv', 'das', 'dds', 'dods', 'html', 'json', 'jsona', 'jsond', 'nc', 'tab', 'txt', 'zip', 'zip3']
 
-        if filename is not None:
-            csv = requests.get(self.query.replace('.csv', suffix)).text
-            f = open(filename, 'w')
-            f.write(csv)
+        if format in validFormats:
+            if self.buildQuery():
+                suffix = '.' + format
+                if '.' not in filename:
+                    filename += suffix
+
+                if filename is not None:
+                    csv = requests.get(self.query.replace('.csv', suffix)).text
+                    f = open(filename, 'w')
+                    f.write(csv)
+            else:
+                logging.error("Cannot create file. Query was none. Check that dataset is valid.")
+        else:
+            logging.error("Cannot create file. " + format + " is not a valid LaTiS format. Valid formats are: " + str(validFormats))
+            
 
     def clearProjections(self) -> None:
         """Clears the list of projections for the dataset."""
@@ -458,7 +473,11 @@ def __init__(self, latisInstance: LatisInstance, dataset: Dataset):
                 variables = pd.read_json(q)['variable']
                 for i in range(len(variables)):
                     self.properties[variables[i]['id']] = variables[i]
+            else:
+                logging.error("Cannot populate metadata. Query was None.")
         else:
             q = latisInstance.baseUrl + dataset.name + '.jsond?first()'
             if _checkQuery(q):
                 self.properties = pd.read_json(q).iloc[1][0]
+            else:
+                logging.error("Cannot populate metadata. Query was None.")
diff --git a/tests/example.py b/tests/example.py
index 7fe97ea..5b44634 100644
--- a/tests/example.py
+++ b/tests/example.py
@@ -124,9 +124,7 @@ def testErrors():
         'https://lasp.colorado.edu/lisird/latis',
         'cls_radio_flux_f83', 'NUMPY', operations=['time<0'],
         preferVersion2=False)
-
-    print(testLatis2Np)
-
+    
     instance = latis.LatisInstance(
         baseUrl='https://lasp.colorado.edu/lisird/latis',
         latis3=False)
@@ -139,58 +137,21 @@ def testErrors():
         baseUrl='https://lasp.colorado.edu/lis3rd/latis',
         latis3=True)
 
-    print('\nSearch Catalog\n')
-    # 2 - Search Catalog
-    # print(instance.catalog.list) # (Full catalog)
-    print(instancebad.catalog.datasets)
-    print(instancebad.catalog.search('so3rce'))
-
-    print('\nGetting Datasets\n')
-    # 3 - Get dataset objects
     clsRadioFluxF8 = instance.getDataset('cls_radio3_flux_f8')
     clsRadioFluxF15 = instance.getDataset('cls_radi3o_flux_f15')
     sorceMGIndex = instance3.getDataset('sorce_mg3_index')
     clsRadioFluxF107 = instance.getDataset('cls_radio3_flux_absolute_f107')
 
-    print('\nCreating Queries\n')
-    # 4 - Create queries
     clsRadioFluxF15.select(start='0')
     sorceMGIndex.select(start='-19023')
     clsRadioFluxF107.project(['232', '23231']).select(start='A', end='QERWEEWD').select(target='absolute_f107', end='70').operate('formatTime(yyyy.MM.dd)')
-    # clsRadioFluxF107.project(['time','absolute_f107']).operate('formatTime(yyyy.MM.dd)').select(target='absolute_f107', end='70').select(start='1953', end='1954')
-
-    print(clsRadioFluxF8.buildQuery())
-    print(clsRadioFluxF15.buildQuery())
-    print(clsRadioFluxF107.buildQuery())
-
-    print('\nGet Metadata\n')
-    # 5 - Get metadata
-    print(clsRadioFluxF15.metadata.properties)
-    print(clsRadioFluxF8.metadata.properties)
-    print(sorceMGIndex.metadata.properties)
-
-    print('\nGet data\n')
-    # 6 - Get data
-
-    print("clsRadioFluxF15:", clsRadioFluxF15.projections,
-          clsRadioFluxF15.selections, clsRadioFluxF15.operations)
-    print("clsRadioFluxF8:", clsRadioFluxF8.projections,
-          clsRadioFluxF8.selections, clsRadioFluxF8.operations)
-    print("clsRadioFluxF107:", clsRadioFluxF107.projections,
-          clsRadioFluxF107.selections, clsRadioFluxF107.operations)
-    print("sorceMGIndex:", sorceMGIndex.projections,
-          sorceMGIndex.selections, sorceMGIndex.operations)
 
     pandasDF = clsRadioFluxF15.asPandas()
-    print(pandasDF)
     numpy = clsRadioFluxF107.asNumpy()
-    print(numpy)
     mgData = sorceMGIndex.asPandas()
-    print(mgData)
-
-    # 7 - Get file
+    
     clsRadioFluxF15.getFile('cls_radio_flux_f15')
-    clsRadioFluxF15.getFile('cls_radio_flux_f15', 'txt')
+    clsRadioFluxF15.getFile('cls_radio_flux_f15', '3txt')
     clsRadioFluxF15.getFile('cls_radio_flux_f15.data')
 
 testErrors()

From 4896f6955980cff40dad8c0d35ac73cd8ea1f1af Mon Sep 17 00:00:00 2001
From: pie-tras <alex.e.pietras@gmail.com>
Date: Wed, 21 Jun 2023 17:00:00 -0600
Subject: [PATCH 10/12] file and line traceback + formatting fixes

---
 src/latis/client.py | 7 +++++--
 tests/example.py    | 4 ++--
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/latis/client.py b/src/latis/client.py
index 471dc43..b2e2f48 100644
--- a/src/latis/client.py
+++ b/src/latis/client.py
@@ -13,6 +13,8 @@
 
 from typing import List, Dict, Any, Union, Optional
 
+FORMAT = '[%(levelname)s: %(filename)s: %(funcName)s: %(lineno)d]: \n %(message)s \n'
+logging.basicConfig(format=FORMAT)
 
 def _datasetWillUseVersion3(baseUrl: str, dataset: str, preferVersion2: bool) -> bool:
     """Checks which version of LaTiS dataset will be used.
@@ -55,9 +57,9 @@ def _checkQuery(query, expectTextError=True):
     r = requests.get(query)
     if r.status_code > 399:
         if expectTextError:
-            logging.error("Query is invalid: " + query + " returned: " + r.text)
+            logging.error("Query is invalid: " + query + "\r\n Got: " + r.text)
         else:
-            logging.error("Query is invalid: " + query + " returned: " + str(r.status_code))
+            logging.error("Query is invalid: " + query + " Got: " + str(r.status_code))
         return False
     else:
         return True
@@ -144,6 +146,7 @@ class LatisInstance:
     """
 
     def __init__(self, baseUrl: str, latis3: bool):
+
         self.baseUrl: str = baseUrl
         self.latis3: bool = latis3
         self._formatBaseUrl()
diff --git a/tests/example.py b/tests/example.py
index 5b44634..4726a26 100644
--- a/tests/example.py
+++ b/tests/example.py
@@ -149,11 +149,11 @@ def testErrors():
     pandasDF = clsRadioFluxF15.asPandas()
     numpy = clsRadioFluxF107.asNumpy()
     mgData = sorceMGIndex.asPandas()
-    
+
     clsRadioFluxF15.getFile('cls_radio_flux_f15')
     clsRadioFluxF15.getFile('cls_radio_flux_f15', '3txt')
     clsRadioFluxF15.getFile('cls_radio_flux_f15.data')
 
 testErrors()
 # testShortcuts()
-# testCore()
+# testCore()
\ No newline at end of file

From 2651d7b7eb8110f5501dbd7a57a11b1bb24e3d0d Mon Sep 17 00:00:00 2001
From: pie-tras <alex.e.pietras@gmail.com>
Date: Wed, 12 Jul 2023 06:24:59 -0600
Subject: [PATCH 11/12] add _checkQuery docstrings

---
 src/latis/client.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/latis/client.py b/src/latis/client.py
index b2e2f48..9630baf 100644
--- a/src/latis/client.py
+++ b/src/latis/client.py
@@ -54,8 +54,20 @@ def _datasetWillUseVersion3(baseUrl: str, dataset: str, preferVersion2: bool) ->
             return False
 
 def _checkQuery(query, expectTextError=True):
+    """Checks to see if query will return an error from LaTiS.
+
+    Args:
+        query (str): The query to be checked.
+        expectTextError (bool, optional): 
+            Error is expected in request text if True.
+            Error is expected only via a status code if False.
+
+    Returns:
+        bool: True if query has no errors, False otherwise.
+    """
+
     r = requests.get(query)
-    if r.status_code > 399:
+    if r.status_code > 399: # Is an error and not just a nominal status code.
         if expectTextError:
             logging.error("Query is invalid: " + query + "\r\n Got: " + r.text)
         else:
@@ -374,7 +386,7 @@ def buildQuery(self) -> str:
             self.query = self.query + '&' + urllib.parse.quote(o)
 
         if not _checkQuery(self.query):
-            self.query = None
+            self.query = ""
         
         return self.query
 

From c7c66dd8b925d48ff0a8a5d5f69f343db53387db Mon Sep 17 00:00:00 2001
From: pie-tras <alex.e.pietras@gmail.com>
Date: Thu, 20 Jul 2023 20:09:04 -0600
Subject: [PATCH 12/12] Error reporting refactors

---
 src/latis/client.py | 47 ++++++++++++++++++++++++++++++---------------
 tests/example.py    | 29 +++++++++++++++-------------
 2 files changed, 48 insertions(+), 28 deletions(-)

diff --git a/src/latis/client.py b/src/latis/client.py
index 9630baf..6f3edd2 100644
--- a/src/latis/client.py
+++ b/src/latis/client.py
@@ -13,7 +13,7 @@
 
 from typing import List, Dict, Any, Union, Optional
 
-FORMAT = '[%(levelname)s: %(filename)s: %(funcName)s: %(lineno)d]: \n %(message)s \n'
+FORMAT = '[%(levelname)s at %(filename)s in %(funcName)s line %(lineno)d]: %(message)s'
 logging.basicConfig(format=FORMAT)
 
 def _datasetWillUseVersion3(baseUrl: str, dataset: str, preferVersion2: bool) -> bool:
@@ -63,18 +63,21 @@ def _checkQuery(query, expectTextError=True):
             Error is expected only via a status code if False.
 
     Returns:
-        bool: True if query has no errors, False otherwise.
+        (bool, str): 
+            First element True if query has no errors, False otherwise.
+            Second element contains any error info from LaTiS.
     """
 
+    errorText = ""
     r = requests.get(query)
     if r.status_code > 399: # Is an error and not just a nominal status code.
         if expectTextError:
-            logging.error("Query is invalid: " + query + "\r\n Got: " + r.text)
+            errorText = "    Query: " + query + " got:\n    " + "\n    ".join(x for x in r.text.strip().split("\n"))
         else:
-            logging.error("Query is invalid: " + query + " Got: " + str(r.status_code))
-        return False
+            errorText = "    Query: " + query + " got: " + str(r.status_code)
+        return {"valid": False, "errorText": errorText}
     else:
-        return True
+        return {"valid": True, "errorText": errorText}
 
 def data(baseUrl: str, dataset: str, returnType: str,
          projections: Optional[List[str]] = None,
@@ -222,7 +225,9 @@ def __init__(self, latisInstance: LatisInstance):
         self.list: np.ndarray[str, np.dtype[Any]]
 
         if latisInstance.latis3:
-            if _checkQuery(latisInstance.baseUrl, expectTextError=False):
+            queryCheck: dict = _checkQuery(latisInstance.baseUrl, expectTextError=False)
+
+            if queryCheck['valid']:
                 js = requests.get(latisInstance.baseUrl).json()
                 dataset = js['dataset']
                 titles = np.array([k['title'] for k in dataset])
@@ -230,18 +235,21 @@ def __init__(self, latisInstance: LatisInstance):
                 for i in range(len(self.list)):
                     self.datasets[titles[i]] = self.list[i]
             else:
-                logging.error("Cannot populate catalog. Query was None.")
+                logging.error("Cannot populate catalog.\n" + queryCheck["errorText"])
                 self.list = np.array([])
         else:
             q = latisInstance.baseUrl + 'catalog.csv'
-            if _checkQuery(q):
+
+            queryCheck: dict = _checkQuery(q)
+
+            if queryCheck['valid']:
                 df = pd.read_csv(q)
                 names = df['name']
                 self.list = df['accessURL'].to_numpy()
                 for i in range(len(self.list)):
                     self.datasets[names[i]] = self.list[i]
             else:
-                logging.error("Cannot populate catalog. Query was None.")
+                logging.error("Cannot populate catalog.\n" + queryCheck["errorText"])
                 self.list = np.array([])
 
     def search(self, searchTerm: str) -> "np.ndarray":
@@ -385,8 +393,11 @@ def buildQuery(self) -> str:
         for o in self.operations:
             self.query = self.query + '&' + urllib.parse.quote(o)
 
-        if not _checkQuery(self.query):
+        queryCheck: dict = _checkQuery(self.query)
+
+        if not queryCheck['valid']:
             self.query = ""
+            logging.error("Cannot build query\n" + queryCheck["errorText"])
         
         return self.query
 
@@ -484,15 +495,21 @@ def __init__(self, latisInstance: LatisInstance, dataset: Dataset):
 
         if latisInstance.latis3:
             q = latisInstance.baseUrl + dataset.name + '.meta'
-            if _checkQuery(q):
+
+            queryCheck: dict = _checkQuery(q)
+
+            if queryCheck['valid']:
                 variables = pd.read_json(q)['variable']
                 for i in range(len(variables)):
                     self.properties[variables[i]['id']] = variables[i]
             else:
-                logging.error("Cannot populate metadata. Query was None.")
+                logging.error("Cannot populate metadata.\n" + queryCheck["errorText"])
         else:
             q = latisInstance.baseUrl + dataset.name + '.jsond?first()'
-            if _checkQuery(q):
+
+            queryCheck: dict = _checkQuery(q)
+
+            if queryCheck['valid']:
                 self.properties = pd.read_json(q).iloc[1][0]
             else:
-                logging.error("Cannot populate metadata. Query was None.")
+                logging.error("Cannot populate metadata.\n" + queryCheck["errorText"])
diff --git a/tests/example.py b/tests/example.py
index 4726a26..cdef357 100644
--- a/tests/example.py
+++ b/tests/example.py
@@ -120,10 +120,10 @@ def testCore():
     clsRadioFluxF15.getFile('cls_radio_flux_f15.data')
 
 def testErrors():
-    testLatis2Np = latis.data(
-        'https://lasp.colorado.edu/lisird/latis',
-        'cls_radio_flux_f83', 'NUMPY', operations=['time<0'],
-        preferVersion2=False)
+    # testLatis2Np = latis.data(
+    #     'https://lasp.colorado.edu/lisird/latis',
+    #     'cls_radio_flux_f83', 'NUMPY', operations=['time<0'],
+    #     preferVersion2=False)
     
     instance = latis.LatisInstance(
         baseUrl='https://lasp.colorado.edu/lisird/latis',
@@ -137,18 +137,21 @@ def testErrors():
         baseUrl='https://lasp.colorado.edu/lis3rd/latis',
         latis3=True)
 
-    clsRadioFluxF8 = instance.getDataset('cls_radio3_flux_f8')
-    clsRadioFluxF15 = instance.getDataset('cls_radi3o_flux_f15')
-    sorceMGIndex = instance3.getDataset('sorce_mg3_index')
-    clsRadioFluxF107 = instance.getDataset('cls_radio3_flux_absolute_f107')
+    instance.getDataset('cls_radio3_flux_f8')
+    instance.getDataset('cls_radi3o_flux_f15')
+    instance3.getDataset('sorce_mg3_index')
+    instance.getDataset('cls_radio3_flux_absolute_f107')
+    
+    clsRadioFluxF8 = instance.getDataset('cls_radio_flux_f8')
+    clsRadioFluxF15 = instance.getDataset('cls_radio_flux_f15')
+    sorceMGIndex = instance3.getDataset('sorce_mg_index')
+    clsRadioFluxF107 = instance.getDataset('cls_radio_flux_absolute_f107')
 
-    clsRadioFluxF15.select(start='0')
-    sorceMGIndex.select(start='-19023')
+    clsRadioFluxF15.select(start='A')
     clsRadioFluxF107.project(['232', '23231']).select(start='A', end='QERWEEWD').select(target='absolute_f107', end='70').operate('formatTime(yyyy.MM.dd)')
 
-    pandasDF = clsRadioFluxF15.asPandas()
-    numpy = clsRadioFluxF107.asNumpy()
-    mgData = sorceMGIndex.asPandas()
+    print(clsRadioFluxF15.asPandas())
+    print(clsRadioFluxF107.asNumpy())
 
     clsRadioFluxF15.getFile('cls_radio_flux_f15')
     clsRadioFluxF15.getFile('cls_radio_flux_f15', '3txt')