From db25cbacea3893ed6a33292ab0d62f834e23ffbe Mon Sep 17 00:00:00 2001 From: mjb101 <42738256+mjb101@users.noreply.github.com> Date: Fri, 18 Nov 2022 22:16:15 +0000 Subject: [PATCH 1/9] Update decode-ColorNote.py changed tabs to spaces modified path creation for "notes.bin" added offset to variable idx to allow for different doc formats --- decode-ColorNote.py | 284 +++++++++++++++++++++++--------------------- 1 file changed, 151 insertions(+), 133 deletions(-) diff --git a/decode-ColorNote.py b/decode-ColorNote.py index 52c5a18..e85ef02 100755 --- a/decode-ColorNote.py +++ b/decode-ColorNote.py @@ -9,148 +9,166 @@ import glob from datetime import datetime from optparse import OptionParser +from os.path import abspath, dirname class PBEWITHMD5AND128BITAES_CBC_OPENSSL: - def __init__(self, password, salt, iterations): - # Iterations aren't used - (self._key, self._iv) = self._get_derived_key_and_iv(password, salt) - - def _get_derived_key_and_iv(self, password, salt): - """ - Returns tuple of key(16 bytes) and iv(16 bytes) for AES - Logic: - This code is inspired by : - /** - * Generator for PBE derived keys and ivs as usd by OpenSSL. - *

- * The scheme is a simple extension of PKCS 5 V2.0 Scheme 1 using MD5 with an - * iteration count of 1. - *

- */ - public class OpenSSLPBEParametersGenerator - :param password: password used for encryption/decryption - :param salt: salt - :return: (16 bytes dk, 16 bytes iv) - """ - - hasher = MD5.new() - hasher.update(password) - hasher.update(salt) - result = hasher.digest() - key = result - - hasher = MD5.new() - hasher.update(result) - hasher.update(password) - hasher.update(salt) - result = hasher.digest() - iv = result - - # key, iv - return key, iv - - def decrypt(self, data): - encoder = AES.new(self._key, AES.MODE_CBC, self._iv) - return encoder.decrypt(data) + def __init__(self, password, salt, iterations): + # Iterations aren't used + (self._key, self._iv) = self._get_derived_key_and_iv(password, salt) + + def _get_derived_key_and_iv(self, password, salt): + """ + Returns tuple of key(16 bytes) and iv(16 bytes) for AES + Logic: + This code is inspired by : + /** + * Generator for PBE derived keys and ivs as usd by OpenSSL. + *

+ * The scheme is a simple extension of PKCS 5 V2.0 Scheme 1 using MD5 with an + * iteration count of 1. + *

+ */ + public class OpenSSLPBEParametersGenerator + :param password: password used for encryption/decryption + :param salt: salt + :return: (16 bytes dk, 16 bytes iv) + """ + + hasher = MD5.new() + hasher.update(password) + hasher.update(salt) + result = hasher.digest() + key = result + + hasher = MD5.new() + hasher.update(result) + hasher.update(password) + hasher.update(salt) + result = hasher.digest() + iv = result + + # key, iv + return key, iv + + def decrypt(self, data): + encoder = AES.new(self._key, AES.MODE_CBC, self._iv) + return encoder.decrypt(data) class Note: - def __init__(self, json): - self._json = json - def __repr__(self): - return json.dumps(self._json, sort_keys=True, indent=4) - def get_uuid(self): - return self._json['uuid'] - def get_created_date(self): - return datetime.fromtimestamp(self._json['created_date'] / 1000) - def get_minor_modified_date(self): - return datetime.fromtimestamp(self._json['minor_modified_date'] / 1000) - def get_modified_date(self): - return datetime.fromtimestamp(self._json['modified_date'] / 1000) - def is_archived(self): - return self._json['space'] == 16 - def get_title(self): - return self._json['title'] - def get_note(self): - return self._json['note'] + def __init__(self, json): + self._json = json + def __repr__(self): + return json.dumps(self._json, sort_keys=True, indent=4) + def get_uuid(self): + return self._json['uuid'] + def get_created_date(self): + return datetime.fromtimestamp(self._json['created_date'] / 1000) + def get_minor_modified_date(self): + return datetime.fromtimestamp(self._json['minor_modified_date'] / 1000) + def get_modified_date(self): + return datetime.fromtimestamp(self._json['modified_date'] / 1000) + def is_archived(self): + return self._json['space'] == 16 + def get_title(self): + return self._json['title'] + def get_note(self): + return self._json['note'] class NotesSet: - def __init__(self): - self._notes = {} - def has_uuid(self, uuid): - return uuid in self._notes.keys() - def update_if_newer(self, note): - if not self.has_uuid(note.get_uuid()): - self._notes[note.get_uuid()] = note - elif self._notes[note.get_uuid()].get_minor_modified_date() < note.get_minor_modified_date(): - self._notes[note.get_uuid()] = note - #else: - # Nothing to do - def get(self): - for (k,n) in sorted(self._notes.items(), key=lambda item: item[1].get_modified_date()): - #print(n) - yield n + def __init__(self): + self._notes = {} + def has_uuid(self, uuid): + return uuid in self._notes.keys() + def update_if_newer(self, note): + if not self.has_uuid(note.get_uuid()): + self._notes[note.get_uuid()] = note + elif self._notes[note.get_uuid()].get_minor_modified_date() < note.get_minor_modified_date(): + self._notes[note.get_uuid()] = note + #else: + # Nothing to do + def get(self): + for (k,n) in sorted(self._notes.items(), key=lambda item: item[1].get_modified_date()): + #print(n) + yield n ## # MAIN def main(): - _salt = b'ColorNote Fixed Salt' - _iterations = 20 # In fact, not required for derivation - - logger = logging.getLogger() - #logger.setLevel(logging.DEBUG) - - parser = OptionParser() - parser.add_option("-p", "--password", action="store", type="string", - dest="password", default="0000", - help="password for uncrypting backup notes") - parser.add_option("-q", "--quiet", - action="store_false", dest="verbose", default=True, - help="don't print status messages to stdout") - - (options, args) = parser.parse_args() - - if len(args) != 1: - parser.error("ColorNote backup directory is missing") - if not os.path.isdir(args[0]): - parser.error("Argument '{}' is not a directory or doesn't exist".format(args[0])) - - backup_directory = args[0] - - notes = NotesSet() - - decoder = PBEWITHMD5AND128BITAES_CBC_OPENSSL(options.password.encode('utf-8'), _salt, _iterations) - - for bakfile in glob.iglob(os.path.join(backup_directory, '**', '*.doc'), recursive=True): - logging.debug(bakfile) - - doc = open(bakfile, "rb").read() - - decoded_doc = decoder.decrypt(doc[28:]) - - open("/tmp/notes.bin", "wb").write(decoded_doc) - - idx = 0x10 - while idx + 4 < len(decoded_doc): - # File is padded with something like 0f0f0f0f or 0b0b0b0b... - if (decoded_doc[idx] == decoded_doc[idx+1] and - decoded_doc[idx+1] == decoded_doc[idx+2] and - decoded_doc[idx+2] == decoded_doc[idx+3]): - break - (chunk_length,) = struct.unpack(">L", decoded_doc[idx:idx+4]) - logging.debug("Chunk length: {}".format(chunk_length)) - chunk = decoded_doc[idx+4:idx+chunk_length+4] - logging.debug("Chunk: {}".format(chunk)) - json_chunk = json.loads(chunk.decode("utf-8")) - notes.update_if_newer(Note(json_chunk)) - idx += chunk_length + 4 - - for n in notes.get(): - if not n.is_archived(): - print('--------') - logging.debug(n) - print(n.get_title()) - print("Created at {}\t Modified at {}".format(n.get_created_date(), n.get_modified_date())) - print(n.get_note()) + _salt = b'ColorNote Fixed Salt' + _iterations = 20 # In fact, not required for derivation + + logger = logging.getLogger() + #logger.setLevel(logging.DEBUG) + + parser = OptionParser() + parser.add_option("-p", "--password", action="store", type="string", + dest="password", default="0000", + help="password for uncrypting backup notes") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + (options, args) = parser.parse_args() + + if len(args) != 1: + parser.error("ColorNote backup directory is missing") + if not os.path.isdir(args[0]): + parser.error("Argument '{}' is not a directory or doesn't exist".format(args[0])) + + backup_directory = args[0] + + notes = NotesSet() + + decoder = PBEWITHMD5AND128BITAES_CBC_OPENSSL(options.password.encode('utf-8'), _salt, _iterations) + + for bakfile in glob.iglob(os.path.join(backup_directory, '**', '*.doc'), recursive=True): + logging.debug(bakfile) + + doc = open(bakfile, "rb").read() + + decoded_doc = decoder.decrypt(doc[28:]) + + #open("/tmp/notes.bin", "wb").write(decoded_doc) # ORIGINAL + + # -------------------------------------------------------------------NEW + # create output path + directory = dirname(abspath(__file__)) + tmp_path = os.path.join(directory, "tmp") + os.makedirs(tmp_path, exist_ok=True) + open(os.path.join(tmp_path, "notes.bin"), "wb").write(bytes(str(decoded_doc),"utf-8")) + # ---------------------------------------------------------------------- + + # -------------------------------------------------------------------NEW + # locate substring to give start offset = idx + 4 + substring = b'{"_id":1,"title"' + offset = decoded_doc.find(substring) + extract = decoded_doc[offset:offset+len(substring)].decode("utf-8") + logging.debug(f'{offset: <10}: {extract}') + idx = offset - 4 + # ---------------------------------------------------------------------- + + #idx = 0x10 # ORIGINAL + while idx + 4 < len(decoded_doc): + # File is padded with something like 0f0f0f0f or 0b0b0b0b... + if (decoded_doc[idx] == decoded_doc[idx+1] and + decoded_doc[idx+1] == decoded_doc[idx+2] and + decoded_doc[idx+2] == decoded_doc[idx+3]): + break + (chunk_length,) = struct.unpack(">L", decoded_doc[idx:idx+4]) + logging.debug("Chunk length: {}".format(chunk_length)) + chunk = decoded_doc[idx+4:idx+chunk_length+4] + logging.debug("Chunk: {}".format(chunk)) + json_chunk = json.loads(chunk.decode("utf-8")) + notes.update_if_newer(Note(json_chunk)) + idx += chunk_length + 4 + + for n in notes.get(): + if not n.is_archived(): + print('--------') + logging.debug(n) + print(n.get_title()) + print("Created at {}\t Modified at {}".format(n.get_created_date(), n.get_modified_date())) + print(n.get_note()) if __name__ == "__main__": - main() + main() From 92f4711a6a164897e0f194981566b3bb27f3641a Mon Sep 17 00:00:00 2001 From: mjb101 <42738256+mjb101@users.noreply.github.com> Date: Sat, 19 Nov 2022 10:03:03 +0000 Subject: [PATCH 2/9] Add files via upload modified to write all notes to a csv file fields written may be selected by editing json_keys_select --- decode-ColorNote-CSV.py | 258 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 258 insertions(+) create mode 100644 decode-ColorNote-CSV.py diff --git a/decode-ColorNote-CSV.py b/decode-ColorNote-CSV.py new file mode 100644 index 0000000..79b5a0e --- /dev/null +++ b/decode-ColorNote-CSV.py @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +from Crypto.Cipher import AES +from Crypto.Hash import MD5 +import logging +import binascii +import json +import struct +import os +import glob +from datetime import datetime +from optparse import OptionParser +from os.path import abspath, dirname +# ---------------------------------------------------------------------------NEW +import csv +import traceback +from datetime import timezone +# ------------------------------------------------------------------------------ + +# ---------------------------------------------------------------------------NEW +def get_excel_date(dt): + # Microsoft Excel date is number of days since 1899-12-30 + # Microsoft Excel date = 1 = 1900-01-01 + # seconds per day = 60*60*24 = 86400 + UTC = timezone.utc + dt_python = dt.replace(tzinfo=UTC) + dt_excel_zero = datetime(1899, 12, 30, tzinfo=UTC) + return (dt_python - dt_excel_zero).total_seconds() / 86400 +# ------------------------------------------------------------------------------ + +class PBEWITHMD5AND128BITAES_CBC_OPENSSL: + def __init__(self, password, salt, iterations): + # Iterations aren't used + (self._key, self._iv) = self._get_derived_key_and_iv(password, salt) + + def _get_derived_key_and_iv(self, password, salt): + """ + Returns tuple of key(16 bytes) and iv(16 bytes) for AES + Logic: + This code is inspired by : + /** + * Generator for PBE derived keys and ivs as usd by OpenSSL. + *

+ * The scheme is a simple extension of PKCS 5 V2.0 Scheme 1 using MD5 with an + * iteration count of 1. + *

+ */ + public class OpenSSLPBEParametersGenerator + :param password: password used for encryption/decryption + :param salt: salt + :return: (16 bytes dk, 16 bytes iv) + """ + + hasher = MD5.new() + hasher.update(password) + hasher.update(salt) + result = hasher.digest() + key = result + + hasher = MD5.new() + hasher.update(result) + hasher.update(password) + hasher.update(salt) + result = hasher.digest() + iv = result + + # key, iv + return key, iv + + def decrypt(self, data): + encoder = AES.new(self._key, AES.MODE_CBC, self._iv) + return encoder.decrypt(data) + +class Note: + def __init__(self, json): + self._json = json + def __repr__(self): + return json.dumps(self._json, sort_keys=True, indent=4) + def get_uuid(self): + return self._json['uuid'] + def get_created_date(self): + return datetime.fromtimestamp(self._json['created_date'] / 1000) + def get_minor_modified_date(self): + return datetime.fromtimestamp(self._json['minor_modified_date'] / 1000) + def get_modified_date(self): + return datetime.fromtimestamp(self._json['modified_date'] / 1000) + def is_archived(self): + return self._json['space'] == 16 + def get_title(self): + return self._json['title'] + def get_note(self): + return self._json['note'] + +class NotesSet: + def __init__(self): + self._notes = {} + def has_uuid(self, uuid): + return uuid in self._notes.keys() + def update_if_newer(self, note): + if not self.has_uuid(note.get_uuid()): + self._notes[note.get_uuid()] = note + elif self._notes[note.get_uuid()].get_minor_modified_date() < note.get_minor_modified_date(): + self._notes[note.get_uuid()] = note + #else: + # Nothing to do + def get(self): + for (k,n) in sorted(self._notes.items(), key=lambda item: item[1].get_modified_date()): + #print(n) + yield n + +## +# MAIN +def main(): + # -----------------------------------------------------------------------NEW + out_name = "ColorNote_backup" + out_extn = ".csv" + out_sep = "_" + json_keys = ["_id", "account_id", "active_state", "color_index", + "created_date", "dirty", "encrypted", "folder_id", + "importance", "latitude", "longitude", "minor_modified_date", + "modified_date", "note", "note_ext", "note_type", + "reminder_base", "reminder_date", "reminder_duration", + "reminder_last", "reminder_option", "reminder_repeat", + "reminder_repeat_ends", "reminder_type", "revision", "space", + "staged", "status", "tags", "title", "type", "uuid"] + json_keys_select = ["_id", "color_index", "created_date", + "minor_modified_date", "modified_date", "note", + "revision", "title"] + # -------------------------------------------------------------------------- + + _salt = b'ColorNote Fixed Salt' + _iterations = 20 # In fact, not required for derivation + + logger = logging.getLogger() + #logger.setLevel(logging.DEBUG) + + parser = OptionParser() + parser.add_option("-p", "--password", action="store", type="string", + dest="password", default="0000", + help="password for uncrypting backup notes") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + (options, args) = parser.parse_args() + + if len(args) != 1: + parser.error("ColorNote backup directory is missing") + if not os.path.isdir(args[0]): + parser.error("Argument '{}' is not a directory or doesn't exist".format(args[0])) + + backup_directory = args[0] + + notes = NotesSet() + + decoder = PBEWITHMD5AND128BITAES_CBC_OPENSSL(options.password.encode('utf-8'), _salt, _iterations) + + for bakfile in glob.iglob(os.path.join(backup_directory, '**', '*.doc'), recursive=True): + logging.debug(bakfile) + + doc = open(bakfile, "rb").read() + + decoded_doc = decoder.decrypt(doc[28:]) + + #open("/tmp/notes.bin", "wb").write(decoded_doc) # ORIGINAL + + # -------------------------------------------------------------------NEW + # create output path + directory = dirname(abspath(__file__)) + tmp_path = os.path.join(directory, "tmp") + os.makedirs(tmp_path, exist_ok=True) + open(os.path.join(tmp_path, "notes.bin"), "wb").write(bytes(str(decoded_doc),"utf-8")) + # ---------------------------------------------------------------------- + + # -------------------------------------------------------------------NEW + # locate substring to give start offset = idx + 4 + substring = b'{"_id":1,"title"' + offset = decoded_doc.find(substring) + extract = decoded_doc[offset:offset+len(substring)].decode("utf-8") + logging.debug(f'{offset: <10}: {extract}') + idx = offset - 4 + # ---------------------------------------------------------------------- + + #idx = 0x10 # ORIGINAL + while idx + 4 < len(decoded_doc): + # File is padded with something like 0f0f0f0f or 0b0b0b0b... + if (decoded_doc[idx] == decoded_doc[idx+1] and + decoded_doc[idx+1] == decoded_doc[idx+2] and + decoded_doc[idx+2] == decoded_doc[idx+3]): + break + (chunk_length,) = struct.unpack(">L", decoded_doc[idx:idx+4]) + logging.debug("Chunk length: {}".format(chunk_length)) + chunk = decoded_doc[idx+4:idx+chunk_length+4] + logging.debug("Chunk: {}".format(chunk)) + json_chunk = json.loads(chunk.decode("utf-8")) + notes.update_if_newer(Note(json_chunk)) + idx += chunk_length + 4 + + #for n in notes.get(): + # if not n.is_archived(): + # print('--------') + # logging.debug(n) + # print(n.get_title()) + # print("Created at {}\t Modified at {}".format(n.get_created_date(), n.get_modified_date())) + # print(n.get_note()) + + # -----------------------------------------------------------------------NEW + dtn = datetime.now() + dtymdhms = dtn.strftime("%Y%m%d_%H%M%S") + dtymd = dtn.strftime("%Y%m%d") + file_path = out_name + out_sep + dtymdhms + out_extn + #with open(file_path, 'a', newline='', encoding='iso-8859-1') as out_file: + with open(file_path, 'a', newline='', encoding='utf-8') as out_file: + csvwriter = csv.writer(out_file, delimiter=',') + + # write headers + #for n in notes.get(): + # njson = json.loads(str(n)) + # break + #csvwriter.writerow(njson.keys()) + #csvwriter.writerow(json_keys) + csvwriter.writerow(json_keys_select) + + for n in notes.get(): + if not n.is_archived(): + print("-"*50) + logging.debug(n) + print(n.get_title()) + print("Created at {}\t Modified at {}".format(n.get_created_date(), n.get_modified_date())) + print(n.get_note()) + njson = json.loads(str(n)) + row = [] + for key in json_keys_select: + match key: + case "created_date": + #value = n.get_created_date().isoformat() + #value = n.get_created_date().strftime("%Y-%m-%d %H:%M:%S.%f") + value = get_excel_date(n.get_created_date()) + case "minor_modified_date": + #value = n.get_minor_modified_date().isoformat() + #value = n.get_minor_modified_date().strftime("%Y-%m-%d %H:%M:%S.%f") + value = get_excel_date(n.get_minor_modified_date()) + case "modified_date": + #value = n.get_modified_date().isoformat() + #value = n.get_modified_date().strftime("%Y-%m-%d %H:%M:%S.%f") + value = get_excel_date(n.get_modified_date()) + case _: + value = njson[key] + row.append(value) + csvwriter.writerow(row) + #csvwriter.writerow(njson.values()) + logging.debug(njson) + logging.debug(njson.keys()) + logging.debug(njson.values()) + logging.debug(njson["note"]) + # -------------------------------------------------------------------------- + +if __name__ == "__main__": + main() From fff906af66ead86130dfba2751a4414cff241cbf Mon Sep 17 00:00:00 2001 From: mjb101 <42738256+mjb101@users.noreply.github.com> Date: Tue, 22 Nov 2022 14:44:12 +0000 Subject: [PATCH 3/9] Add files via upload modified to write all notes to an html table table rows are color coded as per ColorNote table may be sorted by clicking on column headers requires html template file to be in same directory as python script requires that "color_index" be included in json_keys_select --- decode-ColorNote-CSV-HTML-TEMPLATE.html | 207 ++++++++++++++ decode-ColorNote-CSV-HTML.py | 349 ++++++++++++++++++++++++ 2 files changed, 556 insertions(+) create mode 100644 decode-ColorNote-CSV-HTML-TEMPLATE.html create mode 100644 decode-ColorNote-CSV-HTML.py diff --git a/decode-ColorNote-CSV-HTML-TEMPLATE.html b/decode-ColorNote-CSV-HTML-TEMPLATE.html new file mode 100644 index 0000000..fa5b1c5 --- /dev/null +++ b/decode-ColorNote-CSV-HTML-TEMPLATE.html @@ -0,0 +1,207 @@ + + + + + + + + ColorNote Table + + + + +

+ + +
+COLORNOTE_TABLE_PLACEHOLDER + + + + + diff --git a/decode-ColorNote-CSV-HTML.py b/decode-ColorNote-CSV-HTML.py new file mode 100644 index 0000000..4591135 --- /dev/null +++ b/decode-ColorNote-CSV-HTML.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +from Crypto.Cipher import AES +from Crypto.Hash import MD5 +import logging +import binascii +import json +import struct +import os +import glob +from datetime import datetime +from optparse import OptionParser +from os.path import abspath, dirname +# ---------------------------------------------------------------------------NEW +import csv +import traceback +from datetime import timezone +# ------------------------------------------------------------------------------ + +# ---------------------------------------------------------------------------NEW +def get_excel_date(dt): + # Microsoft Excel date is number of days since 1899-12-30 + # Microsoft Excel date = 1 = 1900-01-01 + # seconds per day = 60*60*24 = 86400 + UTC = timezone.utc + dt_python = dt.replace(tzinfo=UTC) + dt_excel_zero = datetime(1899, 12, 30, tzinfo=UTC) + return (dt_python - dt_excel_zero).total_seconds() / 86400 +# ------------------------------------------------------------------------------ + +# ---------------------------------------------------------------------------NEW +def get_html_table(id, caption, list_head, list_data, datetime, indent): + # aligns numbers right for all key values except "note" and "title" + # with option to indent tags + tab = " " * indent if indent else "" + index_note = list_head.index("note") + index_title = list_head.index("title") + list = [] + list.append('') + list.append(tab * 1 + '') + list.append(tab * 1 + '') + list.append(tab * 2 + '') + list.append(tab * 3 + '') + list.append(tab * 2 + '') + list.append(tab * 1 + '') + list.append(tab * 1 + '') + for lrow in list_data: + list.append(tab * 2 + '') + # ---------------------------------------------------------------------- + list_row = [] + for index, item in enumerate(lrow): + if (index != index_note) and (index != index_title): + try: + tmp = int(item) + list_row.append('') + except: + list_row.append('') + else: + list_row.append('') + row = "".join(list_row) + list.append(tab * 3 + row) + # ---------------------------------------------------------------------- + list.append(tab * 2 + '') + list.append(tab * 1 + '') + list.append(tab * 1 + '') + list.append(tab * 2 + '') + list.append(tab * 3 + '') + list.append(tab * 2 + '') + list.append(tab * 1 + '') + list.append('
' + caption + '
' + ''.join(str(x) for x in list_head) + '
' + str(item) + '' + str(item) + '' + str(item) + '
created: ' + datetime + '
') + return list +# ------------------------------------------------------------------------------ + +class PBEWITHMD5AND128BITAES_CBC_OPENSSL: + def __init__(self, password, salt, iterations): + # Iterations aren't used + (self._key, self._iv) = self._get_derived_key_and_iv(password, salt) + + def _get_derived_key_and_iv(self, password, salt): + """ + Returns tuple of key(16 bytes) and iv(16 bytes) for AES + Logic: + This code is inspired by : + /** + * Generator for PBE derived keys and ivs as usd by OpenSSL. + *

+ * The scheme is a simple extension of PKCS 5 V2.0 Scheme 1 using MD5 with an + * iteration count of 1. + *

+ */ + public class OpenSSLPBEParametersGenerator + :param password: password used for encryption/decryption + :param salt: salt + :return: (16 bytes dk, 16 bytes iv) + """ + + hasher = MD5.new() + hasher.update(password) + hasher.update(salt) + result = hasher.digest() + key = result + + hasher = MD5.new() + hasher.update(result) + hasher.update(password) + hasher.update(salt) + result = hasher.digest() + iv = result + + # key, iv + return key, iv + + def decrypt(self, data): + encoder = AES.new(self._key, AES.MODE_CBC, self._iv) + return encoder.decrypt(data) + +class Note: + def __init__(self, json): + self._json = json + def __repr__(self): + return json.dumps(self._json, sort_keys=True, indent=4) + def get_uuid(self): + return self._json['uuid'] + def get_created_date(self): + return datetime.fromtimestamp(self._json['created_date'] / 1000) + def get_minor_modified_date(self): + return datetime.fromtimestamp(self._json['minor_modified_date'] / 1000) + def get_modified_date(self): + return datetime.fromtimestamp(self._json['modified_date'] / 1000) + def is_archived(self): + return self._json['space'] == 16 + def get_title(self): + return self._json['title'] + def get_note(self): + return self._json['note'] + +class NotesSet: + def __init__(self): + self._notes = {} + def has_uuid(self, uuid): + return uuid in self._notes.keys() + def update_if_newer(self, note): + if not self.has_uuid(note.get_uuid()): + self._notes[note.get_uuid()] = note + elif self._notes[note.get_uuid()].get_minor_modified_date() < note.get_minor_modified_date(): + self._notes[note.get_uuid()] = note + #else: + # Nothing to do + def get(self): + for (k,n) in sorted(self._notes.items(), key=lambda item: item[1].get_modified_date()): + #print(n) + yield n + +## +# MAIN +def main(): + # -----------------------------------------------------------------------NEW + # for default_backup_path and default_tmp_path, either assign null string "" + # to use user 'Documents' folder or specify full path, for example: + # backup null string -> r"C:\\Documents\backup" + # backup full path -> r"C:\\backup" + # tmp null string -> r"C:\\Documents\tmp" + # tmp full path -> r"C:\\tmp" + default_backup_path = "" + default_tmp_path = "" + html_template = r"decode-ColorNote-CSV-HTML-TEMPLATE.html" + out_name_csv = "ColorNote_backup" + out_extn_csv = ".csv" + out_sep_csv = "_" + out_name_html = "ColorNote_backup" + out_name_html_tab = "ColorNote_backup_tabulate" + out_extn_html = ".html" + out_sep_html = "_" + json_keys_select = ["_id", "color_index", "created_date", + "minor_modified_date", "modified_date", "note", + "revision", "title"] + # json_keys_select must include key: "color_index" - for html table creation + # json keys available: + # "_id", "account_id", "active_state", "color_index", + # "created_date", "dirty", "encrypted", "folder_id", + # "importance", "latitude", "longitude", "minor_modified_date", + # "modified_date", "note", "note_ext", "note_type", + # "reminder_base", "reminder_date", "reminder_duration", + # "reminder_last", "reminder_option", "reminder_repeat", + # "reminder_repeat_ends", "reminder_type", "revision", "space", + # "staged", "status", "tags", "title", "type", "uuid" + # -------------------------------------------------------------------------- + + _salt = b'ColorNote Fixed Salt' + _iterations = 20 # In fact, not required for derivation + + logger = logging.getLogger() + #logger.setLevel(logging.DEBUG) + + parser = OptionParser() + parser.add_option("-p", "--password", action="store", type="string", + dest="password", default="0000", + help="password for uncrypting backup notes") + parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + + (options, args) = parser.parse_args() + + if len(args) != 1: + parser.error("ColorNote backup directory is missing") + if not os.path.isdir(args[0]): + parser.error("Argument '{}' is not a directory or doesn't exist".format(args[0])) + + backup_directory = args[0] + + notes = NotesSet() + + decoder = PBEWITHMD5AND128BITAES_CBC_OPENSSL(options.password.encode('utf-8'), _salt, _iterations) + + for bakfile in glob.iglob(os.path.join(backup_directory, '**', '*.doc'), recursive=True): + logging.debug(bakfile) + + doc = open(bakfile, "rb").read() + + decoded_doc = decoder.decrypt(doc[28:]) + + #open("/tmp/notes.bin", "wb").write(decoded_doc) # ORIGINAL + + # -------------------------------------------------------------------NEW + # create output path + directory = dirname(abspath(__file__)) + tmp_path = os.path.join(directory, "tmp") + os.makedirs(tmp_path, exist_ok=True) + open(os.path.join(tmp_path, "notes.bin"), "wb").write(bytes(str(decoded_doc),"utf-8")) + # ---------------------------------------------------------------------- + + # -------------------------------------------------------------------NEW + # locate substring to give start offset = idx + 4 + substring = b'{"_id":1,"title"' + offset = decoded_doc.find(substring) + extract = decoded_doc[offset:offset+len(substring)].decode("utf-8") + logging.debug(f"{offset: <10}: {extract}") + idx = offset - 4 + # ---------------------------------------------------------------------- + + #idx = 0x10 # ORIGINAL + while idx + 4 < len(decoded_doc): + # File is padded with something like 0f0f0f0f or 0b0b0b0b... + if (decoded_doc[idx] == decoded_doc[idx+1] and + decoded_doc[idx+1] == decoded_doc[idx+2] and + decoded_doc[idx+2] == decoded_doc[idx+3]): + break + (chunk_length,) = struct.unpack(">L", decoded_doc[idx:idx+4]) + logging.debug("Chunk length: {}".format(chunk_length)) + chunk = decoded_doc[idx+4:idx+chunk_length+4] + logging.debug("Chunk: {}".format(chunk)) + json_chunk = json.loads(chunk.decode("utf-8")) + notes.update_if_newer(Note(json_chunk)) + idx += chunk_length + 4 + + #for n in notes.get(): + # if not n.is_archived(): + # print('--------') + # logging.debug(n) + # print(n.get_title()) + # print("Created at {}\t Modified at {}".format(n.get_created_date(), n.get_modified_date())) + # print(n.get_note()) + + # -----------------------------------------------------------------------NEW + dt = datetime.now() + dtiso = dt.isoformat() + dtymdhms = dt.strftime("%Y%m%d_%H%M%S") + dtymd = dt.strftime("%Y%m%d") + # -------------------------------------------------------------------------- + + # -----------------------------------------------------------------------NEW + file_path = out_name_csv + out_sep_csv + dtymdhms + out_extn_csv + #with open(file_path, "a", newline="", encoding="iso-8859-1") as out_file: + with open(file_path, "a", newline="", encoding="utf-8") as out_file: + csvwriter = csv.writer(out_file, delimiter=",") + + list_html = [] + + # write json keys as headers + json_keys = [] + if json_keys_select: + json_keys = json_keys_select + else: + for n in notes.get(): + njson = json.loads(str(n)) + json_keys = njson.keys() + break + csvwriter.writerow(json_keys) + + # write json values as rows + for n in notes.get(): + if not n.is_archived(): + print("-"*50) + logging.debug(n) + print(n.get_title()) + print("Created at {}\t Modified at {}".format(n.get_created_date(), n.get_modified_date())) + print(n.get_note()) + njson = json.loads(str(n)) + row_csv = [] + row_html = [] + for key in json_keys: + match key: + case "created_date": + value_csv = get_excel_date(n.get_created_date()) + value_html = n.get_created_date().isoformat() + #value = n.get_created_date().strftime("%Y-%m-%d %H:%M:%S.%f") + case "minor_modified_date": + value_csv = get_excel_date(n.get_minor_modified_date()) + value_html = n.get_minor_modified_date().isoformat() + #value = n.get_minor_modified_date().strftime("%Y-%m-%d %H:%M:%S.%f") + case "modified_date": + value_csv = get_excel_date(n.get_modified_date()) + value_html = n.get_modified_date().isoformat() + #value = n.get_modified_date().strftime("%Y-%m-%d %H:%M:%S.%f") + case _: + value_csv = njson[key] + value_html = njson[key] + row_csv.append(value_csv) + row_html.append(value_html) + csvwriter.writerow(row_csv) + #csvwriter.writerow(njson.values()) + list_html.append(row_html) + logging.debug(njson) + logging.debug(njson.keys()) + logging.debug(njson.values()) + logging.debug(njson["note"]) + # -------------------------------------------------------------------------- + + # -----------------------------------------------------------------------NEW + list_table = get_html_table("colornote_table", "ColorNote Table", + json_keys, list_html, dtiso, 0) + search_table_placeholder = "COLORNOTE_TABLE_PLACEHOLDER" + replace_table_text = "\n".join(list_table) + search_index_placeholder = "COLOR_INDEX_COLUMN_PLACEHOLDER" + replace_index_text = str(json_keys.index("color_index")) + # read from source file + with open(html_template, "r") as in_file: + data = in_file.read() + data = data.replace(search_table_placeholder, replace_table_text) + data = data.replace(search_index_placeholder, replace_index_text) + # write to target file + file_path = out_name_html + out_sep_html + dtymdhms + out_extn_html + #with open(file_path, "w", newline="", encoding="iso-8859-1") as out_file: + with open(file_path, "w", newline="", encoding="utf-8") as out_file: + out_file.write(data) + # -------------------------------------------------------------------------- + +if __name__ == "__main__": + main() From af804ddaa9c5bdb4d3f02bac3334ae1ec9f034f1 Mon Sep 17 00:00:00 2001 From: mjb101 <42738256+mjb101@users.noreply.github.com> Date: Wed, 23 Nov 2022 23:28:24 +0000 Subject: [PATCH 4/9] Add files via upload modified to handle '.dat' files (in addition to '.doc' files) by changing magic_offset modified file handling for backup, tmp and output files renamed 'decoded_doc' as 'decoded_backup_file' various minor tweaks --- decode-ColorNote-CSV-HTML.py | 184 ++++++++++++++++++++++++----------- 1 file changed, 128 insertions(+), 56 deletions(-) diff --git a/decode-ColorNote-CSV-HTML.py b/decode-ColorNote-CSV-HTML.py index 4591135..ec7a53b 100644 --- a/decode-ColorNote-CSV-HTML.py +++ b/decode-ColorNote-CSV-HTML.py @@ -14,7 +14,7 @@ import csv import traceback from datetime import timezone -# ------------------------------------------------------------------------------ +# ---------------------------------------------------------------------------END # ---------------------------------------------------------------------------NEW def get_excel_date(dt): @@ -25,7 +25,7 @@ def get_excel_date(dt): dt_python = dt.replace(tzinfo=UTC) dt_excel_zero = datetime(1899, 12, 30, tzinfo=UTC) return (dt_python - dt_excel_zero).total_seconds() / 86400 -# ------------------------------------------------------------------------------ +# ---------------------------------------------------------------------------END # ---------------------------------------------------------------------------NEW def get_html_table(id, caption, list_head, list_data, datetime, indent): @@ -45,12 +45,11 @@ def get_html_table(id, caption, list_head, list_data, datetime, indent): list.append(tab * 1 + '') for lrow in list_data: list.append(tab * 2 + '') - # ---------------------------------------------------------------------- list_row = [] for index, item in enumerate(lrow): if (index != index_note) and (index != index_title): try: - tmp = int(item) + number_test = int(item) list_row.append('' + str(item) + '') except: list_row.append('' + str(item) + '') @@ -58,7 +57,6 @@ def get_html_table(id, caption, list_head, list_data, datetime, indent): list_row.append('' + str(item) + '') row = "".join(list_row) list.append(tab * 3 + row) - # ---------------------------------------------------------------------- list.append(tab * 2 + '') list.append(tab * 1 + '') list.append(tab * 1 + '') @@ -68,7 +66,7 @@ def get_html_table(id, caption, list_head, list_data, datetime, indent): list.append(tab * 1 + '') list.append('') return list -# ------------------------------------------------------------------------------ +# ---------------------------------------------------------------------------END class PBEWITHMD5AND128BITAES_CBC_OPENSSL: def __init__(self, password, salt, iterations): @@ -154,26 +152,43 @@ def get(self): # MAIN def main(): # -----------------------------------------------------------------------NEW - # for default_backup_path and default_tmp_path, either assign null string "" - # to use user 'Documents' folder or specify full path, for example: - # backup null string -> r"C:\\Documents\backup" - # backup full path -> r"C:\\backup" - # tmp null string -> r"C:\\Documents\tmp" - # tmp full path -> r"C:\\tmp" - default_backup_path = "" - default_tmp_path = "" + """ + If the ColorNote backup directory is not given on the command line, the + script will search in directory 'backup_dirname' using paths as follows: + 1. specified_backup_path + 2. user document path (if specified_backup_path = "") + Similarly, tmp and output directories 'tmp_dirname' and 'output_dirname' + will be created using paths as follows: + 1. specified_tmp_path + 2. user document path (if specified_tmp_path = "") + and: + 1. specified_output_path + 2. user document path (if specified_output_path = "") + For example: + backup specified -> r"C:\" + backup user -> r"C:\\Documents\" + output specified -> r"C:\" + output user -> r"C:\\Documents\" + tmp specified -> r"C:\" + tmp user -> r"C:\\Documents\" + """ + specified_backup_path = "" + specified_tmp_path = "ColorNote_tmp" + specified_output_path = "ColorNote_output" + backup_dirname = "backup" + tmp_dirname = "ColorNote_tmp" + output_dirname = "ColorNote_output" html_template = r"decode-ColorNote-CSV-HTML-TEMPLATE.html" out_name_csv = "ColorNote_backup" out_extn_csv = ".csv" out_sep_csv = "_" out_name_html = "ColorNote_backup" - out_name_html_tab = "ColorNote_backup_tabulate" out_extn_html = ".html" out_sep_html = "_" - json_keys_select = ["_id", "color_index", "created_date", - "minor_modified_date", "modified_date", "note", - "revision", "title"] - # json_keys_select must include key: "color_index" - for html table creation + # 'json_keys_select' must include key: "color_index" - for html output + json_keys_select = ["_id", "color_index", + "created_date", "minor_modified_date", "modified_date", + "note", "revision", "title"] # json keys available: # "_id", "account_id", "active_state", "color_index", # "created_date", "dirty", "encrypted", "folder_id", @@ -183,7 +198,7 @@ def main(): # "reminder_last", "reminder_option", "reminder_repeat", # "reminder_repeat_ends", "reminder_type", "revision", "space", # "staged", "status", "tags", "title", "type", "uuid" - # -------------------------------------------------------------------------- + # -----------------------------------------------------------------------END _salt = b'ColorNote Fixed Salt' _iterations = 20 # In fact, not required for derivation @@ -201,57 +216,93 @@ def main(): (options, args) = parser.parse_args() + #if len(args) != 1: + # parser.error("ColorNote backup directory is missing") + #if not os.path.isdir(args[0]): + # parser.error("Argument '{}' is not a directory or doesn't exist".format(args[0])) + + # -----------------------------------------------------------------------NEW + # assign generic paths + script_path = dirname(abspath(__file__)) + user_home_path = os.path.expanduser("~") + user_docs_path = os.path.join(user_home_path, "Documents") + # -----------------------------------------------------------------------END + + # -----------------------------------------------------------------------NEW + # check backup path + user_backup_path = os.path.join(user_docs_path, backup_dirname) + user_backup_path = user_backup_path if os.path.isdir(user_backup_path) else "" if len(args) != 1: - parser.error("ColorNote backup directory is missing") - if not os.path.isdir(args[0]): - parser.error("Argument '{}' is not a directory or doesn't exist".format(args[0])) + backup_directory = specified_backup_path if os.path.isdir(specified_backup_path) else user_backup_path + else: + backup_directory = args[0] + if not os.path.isdir(backup_directory): + parser.error("Argument '{}' is not a directory or doesn't exist".format(backup_directory)) + # -----------------------------------------------------------------------END - backup_directory = args[0] + #backup_directory = args[0] # ORIGINAL notes = NotesSet() decoder = PBEWITHMD5AND128BITAES_CBC_OPENSSL(options.password.encode('utf-8'), _salt, _iterations) - for bakfile in glob.iglob(os.path.join(backup_directory, '**', '*.doc'), recursive=True): + # ----------------------------------------------------------MODIFIED AND NEW + # renamed 'decoded_doc' as 'decoded_backup_file' + backup_files = [] + for type in ("*.dat", "*.doc"): + backup_files.extend(glob.iglob(os.path.join(backup_directory, "**", type), recursive=True)) + logging.debug(backup_files) + for bakfile in backup_files: + #for bakfile in glob.iglob(os.path.join(backup_directory, '**', '*.dat'), recursive=True): # ORIGINAL logging.debug(bakfile) - doc = open(bakfile, "rb").read() + backup_file = open(bakfile, "rb").read() - decoded_doc = decoder.decrypt(doc[28:]) + # handle file types + match os.path.splitext(bakfile)[1].lower(): + case ".dat": + magic_offset = 0 + case ".doc": + magic_offset = 28 + case _: + print("Backup file type not recognised. Require '.dat' or '.doc'.") + Exit + + decoded_backup_file = decoder.decrypt(backup_file[magic_offset:]) #open("/tmp/notes.bin", "wb").write(decoded_doc) # ORIGINAL - # -------------------------------------------------------------------NEW - # create output path - directory = dirname(abspath(__file__)) - tmp_path = os.path.join(directory, "tmp") - os.makedirs(tmp_path, exist_ok=True) - open(os.path.join(tmp_path, "notes.bin"), "wb").write(bytes(str(decoded_doc),"utf-8")) - # ---------------------------------------------------------------------- + # create tmp path + user_tmp_path = os.path.join(user_docs_path, tmp_dirname) + tmp_directory = specified_tmp_path if specified_tmp_path else user_tmp_path + os.makedirs(tmp_directory, exist_ok=True) + + # write to tmp file + open(os.path.join(tmp_directory, "notes.bin"), "wb").write(bytes(str(decoded_backup_file), "utf-8")) + #open(os.path.join(tmp_directory, "notes.bin"), "wb").write(decoded_backup_file) - # -------------------------------------------------------------------NEW # locate substring to give start offset = idx + 4 substring = b'{"_id":1,"title"' - offset = decoded_doc.find(substring) - extract = decoded_doc[offset:offset+len(substring)].decode("utf-8") + offset = decoded_backup_file.find(substring) + extract = decoded_backup_file[offset:offset+len(substring)].decode("utf-8") logging.debug(f"{offset: <10}: {extract}") idx = offset - 4 - # ---------------------------------------------------------------------- #idx = 0x10 # ORIGINAL - while idx + 4 < len(decoded_doc): + while idx + 4 < len(decoded_backup_file): # File is padded with something like 0f0f0f0f or 0b0b0b0b... - if (decoded_doc[idx] == decoded_doc[idx+1] and - decoded_doc[idx+1] == decoded_doc[idx+2] and - decoded_doc[idx+2] == decoded_doc[idx+3]): + if (decoded_backup_file[idx] == decoded_backup_file[idx+1] and + decoded_backup_file[idx+1] == decoded_backup_file[idx+2] and + decoded_backup_file[idx+2] == decoded_backup_file[idx+3]): break - (chunk_length,) = struct.unpack(">L", decoded_doc[idx:idx+4]) + (chunk_length,) = struct.unpack(">L", decoded_backup_file[idx:idx+4]) logging.debug("Chunk length: {}".format(chunk_length)) - chunk = decoded_doc[idx+4:idx+chunk_length+4] + chunk = decoded_backup_file[idx+4:idx+chunk_length+4] logging.debug("Chunk: {}".format(chunk)) json_chunk = json.loads(chunk.decode("utf-8")) notes.update_if_newer(Note(json_chunk)) idx += chunk_length + 4 + # -----------------------------------------------------------------------END #for n in notes.get(): # if not n.is_archived(): @@ -266,12 +317,19 @@ def main(): dtiso = dt.isoformat() dtymdhms = dt.strftime("%Y%m%d_%H%M%S") dtymd = dt.strftime("%Y%m%d") - # -------------------------------------------------------------------------- + # -----------------------------------------------------------------------END # -----------------------------------------------------------------------NEW - file_path = out_name_csv + out_sep_csv + dtymdhms + out_extn_csv - #with open(file_path, "a", newline="", encoding="iso-8859-1") as out_file: - with open(file_path, "a", newline="", encoding="utf-8") as out_file: + # create output path + user_output_path = os.path.join(user_docs_path, output_dirname) + output_directory = specified_output_path if specified_output_path else user_output_path + os.makedirs(output_directory, exist_ok=True) + + # write to csv file + file_name = out_name_csv + out_sep_csv + dtymdhms + out_extn_csv + output_file = os.path.join(output_directory, file_name) + #with open(output_file, "a", newline="", encoding="iso-8859-1") as out_file: + with open(output_file, "a", newline="", encoding="utf-8") as out_file: csvwriter = csv.writer(out_file, delimiter=",") list_html = [] @@ -324,7 +382,7 @@ def main(): logging.debug(njson.keys()) logging.debug(njson.values()) logging.debug(njson["note"]) - # -------------------------------------------------------------------------- + # -----------------------------------------------------------------------END # -----------------------------------------------------------------------NEW list_table = get_html_table("colornote_table", "ColorNote Table", @@ -333,17 +391,31 @@ def main(): replace_table_text = "\n".join(list_table) search_index_placeholder = "COLOR_INDEX_COLUMN_PLACEHOLDER" replace_index_text = str(json_keys.index("color_index")) - # read from source file + # read from html template file with open(html_template, "r") as in_file: data = in_file.read() data = data.replace(search_table_placeholder, replace_table_text) data = data.replace(search_index_placeholder, replace_index_text) - # write to target file - file_path = out_name_html + out_sep_html + dtymdhms + out_extn_html - #with open(file_path, "w", newline="", encoding="iso-8859-1") as out_file: - with open(file_path, "w", newline="", encoding="utf-8") as out_file: + # write to html file + file_name = out_name_html + out_sep_html + dtymdhms + out_extn_html + output_file = os.path.join(output_directory, file_name) + #with open(output_file, "w", newline="", encoding="iso-8859-1") as out_file: + with open(output_file, "w", newline="", encoding="utf-8") as out_file: out_file.write(data) - # -------------------------------------------------------------------------- + # -----------------------------------------------------------------------END if __name__ == "__main__": - main() + #main() # ORIGINAL + # -----------------------------------------------------------------------NEW + #input() + try: + main() + print("\nFinished.") + except: + print(sys.exc_info()[0]) + print(traceback.format_exc()) + print("\nProgram terminated.") + finally: + print("\nPlease press Enter to exit...", end="") + input() + # -----------------------------------------------------------------------END From 01feba5f398dca7bae0820b7ae0c6332fd53a2e9 Mon Sep 17 00:00:00 2001 From: mjb101 <42738256+mjb101@users.noreply.github.com> Date: Mon, 5 Dec 2022 11:58:16 +0000 Subject: [PATCH 5/9] Update decode-ColorNote-CSV-HTML.py modified to handle '.backup' files added import sys --- decode-ColorNote-CSV-HTML.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/decode-ColorNote-CSV-HTML.py b/decode-ColorNote-CSV-HTML.py index ec7a53b..5bbed09 100644 --- a/decode-ColorNote-CSV-HTML.py +++ b/decode-ColorNote-CSV-HTML.py @@ -9,11 +9,12 @@ import glob from datetime import datetime from optparse import OptionParser -from os.path import abspath, dirname # ---------------------------------------------------------------------------NEW import csv +import sys import traceback from datetime import timezone +from os.path import abspath, dirname # ---------------------------------------------------------------------------END # ---------------------------------------------------------------------------NEW @@ -249,7 +250,7 @@ def main(): # ----------------------------------------------------------MODIFIED AND NEW # renamed 'decoded_doc' as 'decoded_backup_file' backup_files = [] - for type in ("*.dat", "*.doc"): + for type in ("*.backup", "*.dat", "*.doc"): backup_files.extend(glob.iglob(os.path.join(backup_directory, "**", type), recursive=True)) logging.debug(backup_files) for bakfile in backup_files: @@ -260,12 +261,14 @@ def main(): # handle file types match os.path.splitext(bakfile)[1].lower(): + case ".backup": + magic_offset = 28 # 12 also appears to work case ".dat": magic_offset = 0 case ".doc": magic_offset = 28 case _: - print("Backup file type not recognised. Require '.dat' or '.doc'.") + print("Backup file type not recognised. Require '.backup' , '.dat' or '.doc'.") Exit decoded_backup_file = decoder.decrypt(backup_file[magic_offset:]) From 0fc926e1236396f7c9ebb18ac09e602d9431e79e Mon Sep 17 00:00:00 2001 From: mjb101 <42738256+mjb101@users.noreply.github.com> Date: Thu, 8 Dec 2022 07:39:23 +0000 Subject: [PATCH 6/9] macros to highlight rows as per ColorNote colors apply to '.csv' file opened with Microsoft Excel or LibreOffice Calc for usage, read instructions within each macro file --- ...Note-CSV-HTML-CALC-MACRO_ColorNoteRows.bas | 123 ++++++++++++++++++ ...Note-CSV-HTML-EXCEL-MACRO_ColorNoteRows.vb | 115 ++++++++++++++++ 2 files changed, 238 insertions(+) create mode 100644 decode-ColorNote-CSV-HTML-CALC-MACRO_ColorNoteRows.bas create mode 100644 decode-ColorNote-CSV-HTML-EXCEL-MACRO_ColorNoteRows.vb diff --git a/decode-ColorNote-CSV-HTML-CALC-MACRO_ColorNoteRows.bas b/decode-ColorNote-CSV-HTML-CALC-MACRO_ColorNoteRows.bas new file mode 100644 index 0000000..64a6de8 --- /dev/null +++ b/decode-ColorNote-CSV-HTML-CALC-MACRO_ColorNoteRows.bas @@ -0,0 +1,123 @@ +' author: Michael Biggs +' created: 2022-12-08 +' description: LibreOffice Basic macro to highlight rows as per +' ColorNote colors. + + +' A. Using LibreOffice Calc to format csv data: +' 1. open '.csv' document file in LibreOffice Calc +' 2. manually select entire cell range containing dates +' Format->Cells...->Category->Date->Format Code +' type 'YYYY-MM-DD HH:MM:SS' then click 'OK' +' 3. manually select entire populated cell range including headers +' Data->AutoFilter +' 4. with entire populated cell range still selected +' Format->Text->Wrap Text +' 5. to adjust column widths: +' select one or more populated columns, then +' either auto adjust using: +' Format->Columns->Optimal Width...->OK +' or manually adjust using: +' Format->Columns->Width... +' type in number, say '7.5' (inches) then click 'OK' +' 6. to adjust row heights: +' select one or more populated rows, then +' either auto adjust using: +' Format->Rows->Optimal Height...->OK +' or manually adjust using: +' Format->Rows->Height... +' type in number, say '1.5' (inches) then click 'OK' +' 7. save document file as '.ods' +' + + +' B. To enable macros in LibreOffice Calc - will prompt on opening: +' 1. Tools->Options...->Security->Macro Security...-> +' select 'Medium' +' 2. OK->OK +' + + +' C. To add 'ColorNoteRows' macro to LibreOffice Calc document: +' 1. open '.bas' macro file in a text editor +' Edit->Select All +' Edit->Copy +' close '.bas' macro file and text editor +' 2. open '.csv', '.ods' or '.xlsx' document file in LibreOffice Calc +' Tools->Macros->Organize Macros->Basic...->Macros From +' click on file name +' click 'New' to open 'Module1' in LibreOffice Basic editor +' Edit->Select All +' Edit->Cut +' Edit->Paste +' File->Close +' 3. right-click on sheet name +' Sheet Events...->Macro... +' click '+' next to file name to reveal 'Standard' +' click '+' next to 'Standard' to reveal 'Module1' +' click on 'Module1' then macro name that appears opposite +' 4. OK->OK +' 5. click '+' next to sheet name (to add another sheet) +' click on original sheet name (to activate sheet and run macro) +' 6. save document file as '.ods' + + +'------------------------------------------------------------------------------- +Public Sub ColorNoteRows() + ' Color each row based on value of cell in color_index column + ' - mimics colors used by ColorNote + ' - requires correct value for color_index_table_column + ' where first column of table = 1, etc + Dim used_range As Object + Dim first_row As Long + Dim first_column As Long + Dim last_row As Long + Dim last_column As Long + Dim color_index_column As Long + Dim color_index_column_offset As Long + Dim i As Long + + color_index_table_column = 2 ' MAKE SURE THIS IS CORRECT + + sheet = ThisComponent.CurrentController.ActiveSheet + + cursor = sheet.createCursor() + cursor.gotoStartOfUsedArea(False) ' move to cell at start of used area + cursor.gotoEndOfUsedArea(True) ' expand to end of used area + used_range = cursor.RangeAddress ' cursor.RangeAddress is the used range + first_row = used_range.StartRow + first_column = used_range.StartColumn + last_row = used_range.EndRow + last_column = used_range.EndColumn + + color_index_column = first_column + color_index_table_column - 1 + + ' getCellByPosition(X, Y) ie (col, row) + ' getCellRangeByPosition(X1 Y1, X2, Y2) ie (col1, row1, col2, row2) + For i = first_row To last_row + With sheet.getCellRangeByPosition(first_column, i, last_column, i) + Select Case sheet.getCellByPosition(color_index_column, i).Value + Case 1 + .CellBackColor = RGB(255, 230, 233) ' red + Case 2 + .CellBackColor = RGB(255, 235, 216) ' orange + Case 3 + .CellBackColor = RGB(254, 248, 186) ' yellow + Case 4 + .CellBackColor = RGB(229, 248, 220) ' green + Case 5 + .CellBackColor = RGB(232, 233, 254) ' blue + Case 6 + .CellBackColor = RGB(239, 224, 255) ' purple + Case 7 + .CellBackColor = RGB(204, 204, 204) ' black + Case 8 + .CellBackColor = RGB(238, 238, 238) ' grey + Case 9 + .CellBackColor = RGB(255, 255, 255) ' white + Case Else + .CellBackColor = -1 + End Select + End With + Next i +End Sub diff --git a/decode-ColorNote-CSV-HTML-EXCEL-MACRO_ColorNoteRows.vb b/decode-ColorNote-CSV-HTML-EXCEL-MACRO_ColorNoteRows.vb new file mode 100644 index 0000000..813c4a7 --- /dev/null +++ b/decode-ColorNote-CSV-HTML-EXCEL-MACRO_ColorNoteRows.vb @@ -0,0 +1,115 @@ +' author: Michael Biggs +' created: 2022-12-08 +' description: Microsoft Visual Basic macro to highlight rows as per +' ColorNote colors. + + +' A. Using Microsoft Excel to format csv data: +' 1. open '.csv' document file in Microsoft Excel +' 2. manually select entire cell range containing dates +' HOME->Cells->Format->Format Cells->Custom->Type +' type 'yyyy-mm-dd hh:mm:ss' then click 'OK' +' 3. manually select entire populated cell range including headers +' INSERT->Table +' check checkbox 'My table has headers' then click 'OK' +' 4. with entire populated cell range still selected +' HOME->Alignment->Wrap Text +' 5. to adjust column widths: +' select one or more populated columns, then +' either auto adjust using: +' HOME->Cells->Format->Autofit Column Width +' or manually adjust using: +' HOME->Cells->Format->Column Width...->Column width +' type in number, say '75' then click 'OK' +' 6. to adjust row heights: +' select one or more populated rows, then +' either auto adjust using: +' HOME->Cells->Format->Autofit Row Height +' or manually adjust using: +' HOME->Cells->Format->Row Height...->Row height +' type in number, say '15' then click 'OK' +' 7. save document file as '.xlsx' +' + + +' B. To enable macros in Microsoft Excel - will prompt on opening: +' 1. File->Options->Trust Center->Trust Center Settings...->Macro Settings-> +' select 'Disable all macros with notification' +' 2. OK->OK +' + + +' C. To add 'ColorNoteRows' macro to Microsoft Excel document: +' 1. open '.vb' macro file in a text editor +' Edit->Select All +' Edit->Copy +' close '.vb' macro file and text editor +' 2. open '.csv' or '.xlsx' document file in Microsoft Excel +' right-click on sheet name +' click View Code to open Microsoft Visual Basic editor +' Edit->Paste +' File->Close and Return to Microsoft Excel +' 3. click '+' next to sheet name (to add another sheet) +' click on original sheet name (to activate sheet and run macro) +' 4. save document file as '.xlsm' + + +'------------------------------------------------------------------------------- +Private Sub Worksheet_Activate() + Call ColorNoteRows +End Sub + + +'------------------------------------------------------------------------------- +Public Sub ColorNoteRows() + ' Color each row based on value of cell in color_index column + ' - mimics colors used by ColorNote + ' - requires correct value for color_index_table_column + ' where first column of table = 1, etc + Dim used_range As Range + Dim first_row As Long + Dim first_column As Long + Dim last_row As Long + Dim last_column As Long + Dim color_index_column As Long + Dim color_index_column_offset As Long + Dim i As Long + + color_index_table_column = 2 ' MAKE SURE THIS IS CORRECT + + Set used_range = ActiveSheet.UsedRange + + first_row = used_range(1).Row + first_column = used_range(1).Column + last_row = used_range(used_range.Cells.Count).Row + last_column = used_range(used_range.Cells.Count).Column + + color_index_column = first_column + color_index_table_column - 1 + + For i = first_row To last_row + With ActiveSheet.Range(Cells(i, first_column), Cells(i, last_column)) + Select Case Cells(i, color_index_column).Value + Case 1 + .Interior.Color = RGB(255, 230, 233) ' red + Case 2 + .Interior.Color = RGB(255, 235, 216) ' orange + Case 3 + .Interior.Color = RGB(254, 248, 186) ' yellow + Case 4 + .Interior.Color = RGB(229, 248, 220) ' green + Case 5 + .Interior.Color = RGB(232, 233, 254) ' blue + Case 6 + .Interior.Color = RGB(239, 224, 255) ' purple + Case 7 + .Interior.Color = RGB(204, 204, 204) ' black + Case 8 + .Interior.Color = RGB(238, 238, 238) ' grey + Case 9 + .Interior.Color = RGB(255, 255, 255) ' white + Case Else + .Interior.Color = xlNone + End Select + End With + Next i +End Sub From 508376e8e23ebd758216dec3c1d2293d3b8f4cb0 Mon Sep 17 00:00:00 2001 From: mjb101 <42738256+mjb101@users.noreply.github.com> Date: Thu, 8 Dec 2022 08:50:29 +0000 Subject: [PATCH 7/9] Update decode-ColorNote-CSV-HTML-CALC-MACRO_ColorNoteRows.bas changed tabs to spaces --- decode-ColorNote-CSV-HTML-CALC-MACRO_ColorNoteRows.bas | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/decode-ColorNote-CSV-HTML-CALC-MACRO_ColorNoteRows.bas b/decode-ColorNote-CSV-HTML-CALC-MACRO_ColorNoteRows.bas index 64a6de8..e63e460 100644 --- a/decode-ColorNote-CSV-HTML-CALC-MACRO_ColorNoteRows.bas +++ b/decode-ColorNote-CSV-HTML-CALC-MACRO_ColorNoteRows.bas @@ -79,16 +79,16 @@ Public Sub ColorNoteRows() color_index_table_column = 2 ' MAKE SURE THIS IS CORRECT - sheet = ThisComponent.CurrentController.ActiveSheet + sheet = ThisComponent.CurrentController.ActiveSheet - cursor = sheet.createCursor() - cursor.gotoStartOfUsedArea(False) ' move to cell at start of used area - cursor.gotoEndOfUsedArea(True) ' expand to end of used area + cursor = sheet.createCursor() + cursor.gotoStartOfUsedArea(False) ' move to cell at start of used area + cursor.gotoEndOfUsedArea(True) ' expand to end of used area used_range = cursor.RangeAddress ' cursor.RangeAddress is the used range first_row = used_range.StartRow first_column = used_range.StartColumn last_row = used_range.EndRow - last_column = used_range.EndColumn + last_column = used_range.EndColumn color_index_column = first_column + color_index_table_column - 1 From a72354ecabf2f5150fff853a6bc6de2f6458480a Mon Sep 17 00:00:00 2001 From: mjb101 <42738256+mjb101@users.noreply.github.com> Date: Sun, 26 Feb 2023 15:56:38 +0000 Subject: [PATCH 8/9] Update decode-ColorNote-CSV-HTML.py code correction to define 'json_keys' as a list for case when 'json_keys_select' is specified as an empty list --- decode-ColorNote-CSV-HTML.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/decode-ColorNote-CSV-HTML.py b/decode-ColorNote-CSV-HTML.py index 5bbed09..0406473 100644 --- a/decode-ColorNote-CSV-HTML.py +++ b/decode-ColorNote-CSV-HTML.py @@ -187,6 +187,8 @@ def main(): out_extn_html = ".html" out_sep_html = "_" # 'json_keys_select' must include key: "color_index" - for html output + # define 'json_keys_select' as empty list to use all available keys + json_keys_select = [] # empty list json_keys_select = ["_id", "color_index", "created_date", "minor_modified_date", "modified_date", "note", "revision", "title"] @@ -344,7 +346,7 @@ def main(): else: for n in notes.get(): njson = json.loads(str(n)) - json_keys = njson.keys() + json_keys = list(njson.keys()) break csvwriter.writerow(json_keys) From a1f1e3267c2f5db6d647b1d029ba102fe64152e2 Mon Sep 17 00:00:00 2001 From: mjb101 <42738256+mjb101@users.noreply.github.com> Date: Mon, 27 Feb 2023 10:55:21 +0000 Subject: [PATCH 9/9] Update decode-ColorNote-CSV-HTML.py modified to correct handling of '.dat' files by fixing value of 'idx' various minor tweaks --- decode-ColorNote-CSV-HTML.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/decode-ColorNote-CSV-HTML.py b/decode-ColorNote-CSV-HTML.py index 0406473..5114730 100644 --- a/decode-ColorNote-CSV-HTML.py +++ b/decode-ColorNote-CSV-HTML.py @@ -186,8 +186,10 @@ def main(): out_name_html = "ColorNote_backup" out_extn_html = ".html" out_sep_html = "_" + + # define 'json_keys_select' + # define as empty list to use all available keys # 'json_keys_select' must include key: "color_index" - for html output - # define 'json_keys_select' as empty list to use all available keys json_keys_select = [] # empty list json_keys_select = ["_id", "color_index", "created_date", "minor_modified_date", "modified_date", @@ -261,9 +263,11 @@ def main(): backup_file = open(bakfile, "rb").read() - # handle file types - match os.path.splitext(bakfile)[1].lower(): - case ".backup": + bakfile_type = os.path.splitext(bakfile)[1].lower() + + # handle file types - assign 'magic_offset' + match bakfile_type: + case ".backup": magic_offset = 28 # 12 also appears to work case ".dat": magic_offset = 0 @@ -291,7 +295,17 @@ def main(): offset = decoded_backup_file.find(substring) extract = decoded_backup_file[offset:offset+len(substring)].decode("utf-8") logging.debug(f"{offset: <10}: {extract}") - idx = offset - 4 + + # handle file types - assign 'idx' + match bakfile_type: + case ".backup": + idx = offset - 4 + case ".dat": + idx = 0x10 + case ".doc": + idx = offset - 4 + case _: + idx = offset - 4 #idx = 0x10 # ORIGINAL while idx + 4 < len(decoded_backup_file): @@ -318,6 +332,7 @@ def main(): # print(n.get_note()) # -----------------------------------------------------------------------NEW + # define date time strings dt = datetime.now() dtiso = dt.isoformat() dtymdhms = dt.strftime("%Y%m%d_%H%M%S")