Add methods to get and set run data

.github/workflows/main.yml

@@ -16,17 +16,17 @@ jobs:
         python-version: ["3.10", "3.11"]
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
-        
+        uses: actions/checkout@v6
+
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
-      
+
       - name: Install dependencies
         run: |
           pip install vcrpy pytest==7.4.2 requests pytest-mock python-documentcloud pytest-xdist pytest-recording python-squarelet
-        
+
       - name: Run pre-recorded tests
         run: |
           make test
@@ -36,17 +36,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v6
 
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v4
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v6
         with:
-          python-version: 3.8
-          
+          python-version: "3.11"
+
       - name: Install dependencies for imports
         run: |
-           pip install python-dateutil requests urllib3 fastjsonschema ratelimit listcrunch pyyaml pytest vcrpy python-squarelet
-           
+          pip install python-dateutil requests urllib3 fastjsonschema ratelimit listcrunch pyyaml pytest vcrpy python-squarelet
+
       - name: Install pylint and black
         run: |
           pip install pylint black

.gitignore

@@ -17,3 +17,4 @@ _build/
 Pipfile
 Pipfile.lock
 .env
+.DS_Store

.isort.cfg

@@ -3,7 +3,6 @@ line_length=88
 force_to_top=
 skip=
 skip_glob=
-not_skip=__init__.py
 known_future_library=__future__
 known_first_party=documentcloud
 indent='    '

.pylintrc

@@ -3,7 +3,4 @@ max-line-length=88
 good-names=i,x1,x2,y1,y2,id
 
 [MESSAGES CONTROL]
-disable=missing-docstring,too-many-ancestors,too-few-public-methods,no-else-return,no-member,attribute-defined-outside-init,similarities,import-outside-toplevel,cyclic-import,no-member,no-else-raise,too-many-instance-attributes,too-many-arguments,ungrouped-imports,useless-object-inheritance,no-else-continue
-
-[DESIGN]
-max-positional-arguments=10
+disable=missing-docstring,too-many-ancestors,too-few-public-methods,no-else-return,no-member,attribute-defined-outside-init,similarities,import-outside-toplevel,cyclic-import,no-member,no-else-raise,too-many-instance-attributes,too-many-arguments,too-many-positional-arguments,ungrouped-imports,useless-object-inheritance,no-else-continue

Makefile

@@ -41,10 +41,10 @@ coverage:
 check:
 	pylint documentcloud
 	black documentcloud
-	isort -rc documentcloud
+	isort documentcloud
 	pylint tests
 	black tests
-	isort -rc tests
+	isort tests
 
 # release a new version of the package to PyPI
 ship:

documentcloud/addon.py

@@ -182,6 +182,26 @@ def upload_file(self, file):
             f"addon_runs/{self.id}/", json={"file_name": file_name}
         )
 
+    def load_run_data(self):
+        "Load persistent data from this run"
+        if not self.id:
+            return {}
+
+        response = self.client.get(f"addon_runs/{self.id}/")
+        response.raise_for_status()
+        return response.json().get("data", {})
+
+    def store_run_data(self, data):
+        "Store persistent data for this run"
+        if not self.id:
+            print("Run ID not set. Try again later or check if something went wrong.")
+            return None
+
+        if not isinstance(data, dict):
+            raise TypeError("Invalid data")
+
+        return self.client.patch(f"addon_runs/{self.id}/", json={"data": data})
+
     def load_event_data(self):
         """Load persistent data for this event"""
         if not self.event_id:

documentcloud/documents.py

@@ -9,6 +9,7 @@
 import re
 import warnings
 from functools import partial
+from urllib.parse import urlparse
 
 # Third Party
 from requests.exceptions import RequestException
@@ -23,11 +24,6 @@
 from .toolbox import grouper, is_url, merge_dicts, requests_retry_session
 from .users import User
 
-try:
-    from urllib.parse import urlparse
-except ImportError:
-    from urlparse import urlparse
-
 logger = logging.getLogger("documentcloud")
 
 IMAGE_SIZES = ["thumbnail", "small", "normal", "large", "xlarge"]

documentcloud/exceptions.py

@@ -2,11 +2,14 @@
 Custom exceptions for python-documentcloud
 """
 
+# Third Party
 # pylint: disable=unused-import
 # Import exceptions from python-squarelet
-from squarelet.exceptions import SquareletError as DocumentCloudError
-from squarelet.exceptions import DuplicateObjectError
-from squarelet.exceptions import CredentialsFailedError
-from squarelet.exceptions import APIError
-from squarelet.exceptions import DoesNotExistError
-from squarelet.exceptions import MultipleObjectsReturnedError
+from squarelet.exceptions import (
+    APIError,
+    CredentialsFailedError,
+    DoesNotExistError,
+    DuplicateObjectError,
+    MultipleObjectsReturnedError,
+    SquareletError as DocumentCloudError,
+)

tests/README.md

@@ -1,29 +1,29 @@
-This library interacts with the API for DocumentCloud.  Running the test suite
+This library interacts with the API for DocumentCloud. Running the test suite
 against the live server provides many challenges - it is slow, will not work
 without an internet connection, and could give false failures for intermitent
-network failures.  To resolve these issues we use VCR.py
-(https://github.com/kevin1024/vcrpy) to record the HTTP requests.  When new
+network failures. To resolve these issues we use VCR.py
+(https://github.com/kevin1024/vcrpy) to record the HTTP requests. When new
 HTTP requests are needed for the tests, they are recorded against a
-localinstance of the server.  Then the tests can be run against the
+localinstance of the server. Then the tests can be run against the
 pre-recorded responses, quickly and without interacting with any other
 software.
 
 To record the HTTP requests, you must have a local dev environment of Squarelet
-and DocumentCloud running.  You can find instructions for those here:
+and DocumentCloud running. You can find instructions for those here:
 https://github.com/MuckRock/squarelet and
-https://github.com/MuckRock/DocumentCloud.  
+https://github.com/MuckRock/DocumentCloud.
 
 You should create a test user locally, with the username `test-user` and password `test-password`.
 
-There are some tests which require the access and refresh tokens to be expired.  To accomodate this, those tests are expected to be run with the local Squarelet instance configured with very short lifetimes for those tokens.  You should record the regular tests, change the settings, run the short tests, then change the settings back.  The settings to change are located in `config/settings/base.py` in the Squarelet code base.  Find the follow lines and uncomment the second two:
+There are some tests which require the access and refresh tokens to be expired. To accomodate this, those tests are expected to be run with the local Squarelet instance configured with very short lifetimes for those tokens. You should record the regular tests, change the settings, run the short tests, then change the settings back. The settings to change are located in `config/settings/base.py` in the Squarelet code base. Find the follow lines and uncomment the second two:
 
 ```
     # These are used for testing token expiration
     # "ACCESS_TOKEN_LIFETIME": timedelta(seconds=2),
     # "REFRESH_TOKEN_LIFETIME": timedelta(seconds=5),
 ```
 
-There is a Makefile included to help run the tests.  The following commands are available:
+There is a Makefile included to help run the tests. The following commands are available:
 
 `test-clean` - This will clean all of the pre-recorded requests for the non-short tests
 
@@ -33,9 +33,9 @@ There is a Makefile included to help run the tests.  The following commands are
 
 `test-create-short` - This will clean the short tests and then run all of them and record the HTTP requests.
 
-`test` - run all tests using the pre-recorded HTTP requests.  If an HTTP request is missing, it will fail.
+`test` - run all tests using the pre-recorded HTTP requests. If an HTTP request is missing, it will fail.
 
-`test-dev` - run all tests using the pre-recorded HTTP requests.  If an HTTP request is missing, record it.
+`test-dev` - run all tests using the pre-recorded HTTP requests. If an HTTP request is missing, record it.
 
 `tox` - run all tests under multiple Python versions using tox.
 
@@ -45,38 +45,40 @@ There is a Makefile included to help run the tests.  The following commands are
 
 `ship` - Release a new version of the library on PyPI.
 
-A normal workflow would be to use `test-create` to create the intial saved requests, or if you want to re-record all of them for some reason.  You would then change the short settings as described above, and run `test-create-short`.  Running `test` should now pass while making no actual HTTP requests.  If you add a new test with a new request, you can run `test-dev` to record just the new request while leaving the existing ones in place.  The saved requests should be checked in to git.
+A normal workflow would be to use `test-create` to create the intial saved requests, or if you want to re-record all of them for some reason. You would then change the short settings as described above, and run `test-create-short`. Running `test` should now pass while making no actual HTTP requests. If you add a new test with a new request, you can run `test-dev` to record just the new request while leaving the existing ones in place. The saved requests should be checked in to git.
 
 ## Troubleshooting
 
-### Token Errors 
+### Token Errors
+
 If you receive a lot of errors that are 405's with E requests.exceptions.HTTPError: 405 Client Error: Method Not Allowed for url: https://dev.squarelet.com/api/token/
-You need to change the BASE_URI and AUTH_URI to https. 
+You need to change the BASE_URI and AUTH_URI to https.
 
 ### SSL Errors
-If you receive a bunch of 500/SSL errors when running the tests, it is likely that your local dev environment doesn't have access to the necessary certificates to authenticate with your local DocumentCloud environment. 
-You can copy the .PEM/.CRT file that is inside the docker container to your local environment and pass this file in so make the SSL errors go away. 
 
-To resolve this you will want to have your local DocumentCloud environment running, including the Django container. You can find the container ID of a running container by running 
-```docker ps```
+If you receive a bunch of 500/SSL errors when running the tests, it is likely that your local dev environment doesn't have access to the necessary certificates to authenticate with your local DocumentCloud environment.
+You can copy the .PEM/.CRT file that is inside the docker container to your local environment and pass this file in so make the SSL errors go away.
+
+To resolve this you will want to have your local DocumentCloud environment running, including the Django container. You can find the container ID of a running container by running
+`docker ps`
 
 Then, to retrieve the certificate run:
-```docker cp container_id_here:/etc/ssl/certs/ca-certificates.crt ~```
-Substitute ~ with the location where you would like to copy the certificate file, as ~ is the home directory. 
+`docker cp container_id_here:/etc/ssl/certs/ca-certificates.crt ~`
+Substitute ~ with the location where you would like to copy the certificate file, as ~ is the home directory.
 
-You can then run the full test-suite by re-recording results  and passing in the necessary certificate like so:
-```REQUESTS_CA_BUNDLE=/path/to/ca-certificates.crt make test-create```
-/path/to should be replaced by the actual location. 
+You can then run the full test-suite by re-recording results and passing in the necessary certificate like so:
+`REQUESTS_CA_BUNDLE=/path/to/ca-certificates.crt make test-create`
+/path/to should be replaced by the actual location.
 
 ### Assertion Errors
 
-If you get a failure for the contributor method, it is because you need to set a full name for the test user within Squarelet. 
+If you get a failure for the contributor method, it is because you need to set a full name for the test user within Squarelet.
 
 If you receive the following failure:
-```assert len(list(all_documents)) > len(list(my_documents.results))```
-It is because you need to have another user created on your local dev environment on Squarelet, have them verified, and have them upload at least one document. 
-This tests asserts that the total sum of documents in your local dev environment is larger than those owned by you. This wouldn't hold true if your test user was the only user who has uploaded a document. 
+`assert len(list(all_documents)) > len(list(my_documents.results))`
+It is because you need to have another user created on your local dev environment on Squarelet, have them verified, and have them upload at least one document.
+This tests asserts that the total sum of documents in your local dev environment is larger than those owned by you. This wouldn't hold true if your test user was the only user who has uploaded a document.
 
 If you receive this similar assertion failure:
-```assert len(all_projects.results) > len(my_projects.results)```
-You will need to have that other user create a project as well. This is to pass this assertion. 
+`assert len(all_projects.results) > len(my_projects.results)`
+You will need to have that other user create a project as well. This is to pass this assertion.