Hotfixes into unstable

.pre-commit-config.yaml

@@ -81,6 +81,15 @@ repos:
           contentcuration/kolibri_public/migrations/0004_auto_20240612_1847.py|
           contentcuration/kolibri_public/migrations/0006_auto_20250417_1516.py|
           )$
+  # Only checks the root Makefile. Extend if nested Makefiles get added.
+  - repo: local
+    hooks:
+      - id: makefile-syntax
+        name: Makefile syntax check
+        entry: make -n
+        language: system
+        files: ^Makefile$
+        pass_filenames: false
   # Always keep black as the final hook so it reformats any other reformatting.
   - repo: https://github.com/python/black
     rev: 20.8b1

Makefile

@@ -39,7 +39,7 @@ migrate:
 # 4) Remove the management command from this `deploy-migrate` recipe
 # 5) Repeat!
 deploy-migrate:
-	python contentcuration/manage.py ensure_versioned_databases_exist & python contentcuration/manage.py create_channel_versions & wait
+	echo "Nothing to do here!"
 
 contentnodegc:
 	python contentcuration/manage.py garbage_collect

contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/components/EditorToolbar.vue

@@ -386,10 +386,14 @@
       // Categories that can overflow (in order of overflow priority)
       const OVERFLOW_CATEGORIES = [
         'insert',
-        'script',
+        // Perseus flavoured markdown does not support super and sub script,
+        // so we disable this for now until we stop using markdown as the primary target
+        // 'script',
         'lists',
         'clearFormat',
-        'align',
+        // Perseus flavoured markdown does not support alignment,
+        // so we disable this for now until we stop using markdown as the primary target
+        // 'align',
         'clipboard',
         'textFormat',
       ];

contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/components/toolbar/MobileFormattingBar.vue

@@ -81,13 +81,17 @@
         :is-active="action.isActive"
         @click="action.handler"
       />
+      <!-- Perseus flavoured markdown does not support alignment,
+        so we disable this for now until we stop using markdown as the primary target
       <ToolbarDivider />
       <ToolbarButton
         :title="alignAction.title"
         :icon="alignAction.icon"
         :is-active="alignAction.isActive"
         @click="alignAction.handler"
-      />
+      /> -->
+      <!-- Perseus flavoured markdown does not support super and sub script,
+        so we disable this for now until we stop using markdown as the primary target
       <ToolbarDivider />
       <ToolbarButton
         v-for="action in scriptActions"
@@ -96,7 +100,7 @@
         :icon="action.icon"
         :is-active="action.isActive"
         @click="action.handler"
-      />
+      /> -->
       <ToolbarDivider />
       <ToolbarButton
         v-for="tool in insertTools"
@@ -138,8 +142,7 @@
         textFormattingToolbar$,
       } = getTipTapEditorStrings();
 
-      const { textActions, listActions, scriptActions, insertTools, alignAction } =
-        useToolbarActions(emit);
+      const { textActions, listActions, insertTools } = useToolbarActions(emit);
 
       const { canIncreaseFormat, canDecreaseFormat, increaseFormat, decreaseFormat } =
         useFormatControls();
@@ -206,9 +209,7 @@
         keyboardOffset,
         textActions,
         listActions,
-        scriptActions,
         insertTools,
-        alignAction,
         toggleToolbar,
         canIncreaseFormat,
         canDecreaseFormat,

contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/composables/useEditor.js

@@ -10,6 +10,7 @@ import { CodeBlockSyntaxHighlight } from '../extensions/CodeBlockSyntaxHighlight
 import { CustomLink } from '../extensions/Link';
 import { Math } from '../extensions/Math';
 import { createCustomMarkdownSerializer } from '../utils/markdownSerializer';
+import { transformPastedHTML } from '../utils/pasteTransform';
 
 export function useEditor() {
   const editor = ref(null);
@@ -42,6 +43,7 @@ export function useEditor() {
           class: 'prose prose-sm sm:prose lg:prose-lg xl:prose-2xl focus:outline-none',
           dir: 'auto',
         },
+        transformPastedHTML: html => transformPastedHTML(html),
       },
       onCreate: () => {
         isReady.value = true;

contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/composables/useToolbarActions.js

@@ -1,6 +1,6 @@
 import { computed, inject } from 'vue';
 import { getTipTapEditorStrings } from '../TipTapEditorStrings';
-import { sanitizePastedHTML } from '../utils/markdown';
+import { transformPastedHTML } from '../utils/pasteTransform';
 
 export function useToolbarActions(emit) {
   const editor = inject('editor', null);
@@ -165,7 +165,7 @@ export function useToolbarActions(emit) {
           if (item.types.includes('text/html')) {
             const htmlBlob = await item.getType('text/html');
             const html = await htmlBlob.text();
-            const cleaned = sanitizePastedHTML(html);
+            const cleaned = transformPastedHTML(html);
 
             editor.value.chain().focus().insertContent(cleaned).run();
             return;

contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdown.js

@@ -39,6 +39,11 @@ export const paramsToImageMd = ({ src, alt, width, height, permanentSrc, textAli
   return `![${alt || ''}](${IMAGE_PLACEHOLDER}/${fileName}${alignSuffix})`;
 };
 
+// --- Underline Translation ---
+// Perseus simple-markdown treats __text__ as <u>; marked treats it as <strong>.
+// Rewrite before marked runs so the round-trip preserves underline.
+export const UNDERLINE_REGEX = /__([\s\S]+?)__(?!_)/g;
+
 // --- Math/Formula Translation ---
 export const MATH_REGEX = /\$\$([^$]+)\$\$/g;
 
@@ -53,67 +58,6 @@ export const paramsToMathMd = ({ latex }) => {
   return `$$${latex || ''}$$`;
 };
 
-export function sanitizePastedHTML(html) {
-  if (!html) return '';
-  // This code ine 55 to 66 is geneted with the help of LLM with the prompt
-  // "Create a function that sanitizes HTML pasted from Microsoft
-  // Word by removing Word-specific tags, styles, and classes while preserving other formatting."
-  let cleaned = html;
-  cleaned = cleaned.replace(/<!--\[if.*?endif\]-->/gis, '');
-  cleaned = cleaned.replace(/<\/?(w|m|o|v):[^>]*>/gis, '');
-  const parser = new DOMParser();
-  const doc = parser.parseFromString(cleaned, 'text/html');
-  doc.querySelectorAll('*').forEach(el => {
-    if (el.hasAttribute('style')) {
-      const style = el.getAttribute('style') || '';
-      const filtered = style
-        .split(';')
-        .map(s => s.trim())
-        .filter(s => s && !s.toLowerCase().startsWith('mso-'))
-        .join('; ');
-      if (filtered) {
-        el.setAttribute('style', filtered);
-      } else {
-        el.removeAttribute('style');
-      }
-    }
-    if (el.hasAttribute('class')) {
-      const cls = el
-        .getAttribute('class')
-        .split(/\s+/)
-        .filter(c => c && !/^Mso/i.test(c))
-        .join(' ');
-      if (cls) {
-        el.setAttribute('class', cls);
-      } else {
-        el.removeAttribute('class');
-      }
-    }
-  });
-  const strikeElements = doc.querySelectorAll('s, strike, del');
-  strikeElements.forEach(el => {
-    const nestedLists = el.querySelectorAll('ul, ol');
-    if (nestedLists.length > 0) {
-      nestedLists.forEach(list => {
-        el.parentNode.insertBefore(list, el.nextSibling);
-      });
-    }
-  });
-  const lists = doc.querySelectorAll('ul, ol');
-  lists.forEach(list => {
-    const items = list.querySelectorAll(':scope > li');
-    items.forEach(item => {
-      const nestedLists = Array.from(item.children).filter(
-        child => child.tagName === 'UL' || child.tagName === 'OL',
-      );
-      nestedLists.forEach(nestedList => {
-        item.appendChild(nestedList);
-      });
-    });
-  });
-  return doc.body.innerHTML;
-}
-
 /**
  * Pre-processes a raw Markdown string to convert custom syntax into HTML tags
  * that Tiptap's extensions can understand. This is our custom "loader".
@@ -151,5 +95,7 @@ export function preprocessMarkdown(markdown) {
     return `<span data-latex="${params.latex}"></span>`;
   });
 
+  processedMarkdown = processedMarkdown.replace(UNDERLINE_REGEX, '<u>$1</u>');
+
   return marked(processedMarkdown);
 }

contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/markdownSerializer.js

@@ -24,7 +24,7 @@ export const createCustomMarkdownSerializer = editor => {
             trimmedText = `*${trimmedText}*`;
             break;
           case 'underline':
-            trimmedText = `<u>${trimmedText}</u>`;
+            trimmedText = `__${trimmedText}__`;
             break;
           case 'strike':
             trimmedText = `~~${trimmedText}~~`;

contentcuration/contentcuration/frontend/shared/views/TipTapEditor/TipTapEditor/utils/pasteTransform.js

@@ -0,0 +1,83 @@
+function stripMsoConditionalComments(html) {
+  return html.replace(/<!--\[if.*?endif\]-->/gis, '');
+}
+
+function stripOfficeNamespacedTags(html) {
+  return html.replace(/<\/?(w|m|o|v):[^>]*>/gis, '');
+}
+
+function filterMsoStyleDeclarations(doc) {
+  doc.querySelectorAll('[style]').forEach(el => {
+    const filtered = el
+      .getAttribute('style')
+      .split(';')
+      .map(s => s.trim())
+      .filter(s => s && !s.toLowerCase().startsWith('mso-'))
+      .join('; ');
+    if (filtered) {
+      el.setAttribute('style', filtered);
+    } else {
+      el.removeAttribute('style');
+    }
+  });
+}
+
+function filterMsoClasses(doc) {
+  doc.querySelectorAll('[class]').forEach(el => {
+    const cls = el
+      .getAttribute('class')
+      .split(/\s+/)
+      .filter(c => c && !/^Mso/i.test(c))
+      .join(' ');
+    if (cls) {
+      el.setAttribute('class', cls);
+    } else {
+      el.removeAttribute('class');
+    }
+  });
+}
+
+function hoistListsOutOfStrike(doc) {
+  doc.querySelectorAll('s, strike, del').forEach(el => {
+    el.querySelectorAll('ul, ol').forEach(list => {
+      el.parentNode.insertBefore(list, el.nextSibling);
+    });
+  });
+}
+
+function reparentNestedListsInLi(doc) {
+  doc.querySelectorAll('ul, ol').forEach(list => {
+    list.querySelectorAll(':scope > li').forEach(item => {
+      Array.from(item.children)
+        .filter(child => child.tagName === 'UL' || child.tagName === 'OL')
+        .forEach(nestedList => item.appendChild(nestedList));
+    });
+  });
+}
+
+function stripImages(doc) {
+  doc.querySelectorAll('img').forEach(el => el.remove());
+}
+
+const STRING_TRANSFORMS = [stripMsoConditionalComments, stripOfficeNamespacedTags];
+
+const DOM_TRANSFORMS = [
+  filterMsoStyleDeclarations,
+  filterMsoClasses,
+  hoistListsOutOfStrike,
+  reparentNestedListsInLi,
+  stripImages,
+];
+
+export function transformPastedHTML(html) {
+  if (!html) return '';
+  let cleaned = html;
+  for (const transform of STRING_TRANSFORMS) {
+    cleaned = transform(cleaned);
+  }
+  const doc = new DOMParser().parseFromString(cleaned, 'text/html');
+  for (const transform of DOM_TRANSFORMS) {
+    transform(doc);
+  }
+  return doc.body.innerHTML;
+}

contentcuration/contentcuration/frontend/shared/views/TipTapEditor/__tests__/markdown.spec.js

@@ -70,7 +70,28 @@ describe('preprocessMarkdown', () => {
     });
   });
 
-  // 4: Standard Markdown Passthrough
+  // 4: Perseus Underline Handling
+  // Perseus's simple-markdown treats __text__ as <u>, but `marked` (CommonMark)
+  // treats it as <strong>. We must rewrite __text__ -> <u>text</u> before marked
+  // sees it, so the round-trip preserves underline instead of turning it into bold.
+  describe('Perseus Underline Handling', () => {
+    it('should rewrite __text__ to <u>text</u> before passing to marked', () => {
+      preprocessMarkdown('This is __underlined__ text.');
+      expect(marked).toHaveBeenCalledWith('This is <u>underlined</u> text.');
+    });
+
+    it('should rewrite multiple __ runs on the same line independently', () => {
+      preprocessMarkdown('__one__ and __two__');
+      expect(marked).toHaveBeenCalledWith('<u>one</u> and <u>two</u>');
+    });
+
+    it('should rewrite __ spanning multiple words', () => {
+      preprocessMarkdown('__multi word underline__');
+      expect(marked).toHaveBeenCalledWith('<u>multi word underline</u>');
+    });
+  });
+
+  // 5: Standard Markdown Passthrough
   describe('Standard Markdown Passthrough', () => {
     it('should pass non-custom syntax through to the marked library', () => {
       const standardMd = 'Here is **bold** and a [link](url).';

contentcuration/contentcuration/frontend/shared/views/TipTapEditor/__tests__/markdownSerializer.spec.js

@@ -209,7 +209,7 @@ describe('createCustomMarkdownSerializer', () => {
       expect(getMarkdown()).toBe('*' + '**bold and italic**' + '*');
     });
 
-    it('should serialize an underlined node with a <u> tag', () => {
+    it('should serialize an underlined node with __ delimiters (Perseus syntax)', () => {
       const docContent = [
         {
           type: 'paragraph',
@@ -224,7 +224,7 @@ describe('createCustomMarkdownSerializer', () => {
       ];
       const mockEditor = createMockEditor(docContent);
       const getMarkdown = createCustomMarkdownSerializer(mockEditor);
-      expect(getMarkdown()).toBe('<u>underlined</u>');
+      expect(getMarkdown()).toBe('__underlined__');
     });
 
     it('should serialize a link node correctly', () => {