impr: support more filetypes for text import (@fehmer)

frontend/__tests__/utils/strings.spec.ts

@@ -590,7 +590,6 @@ describe("string utils", () => {
  });
  });
  });
-
  describe("countChars", () => {
  describe("it should count characters correctly", () => {
  const testCases = [

frontend/package.json

@@ -64,6 +64,7 @@
  "lz-ts": "1.1.2",
  "modern-screenshot": "4.6.8",
  "object-hash": "3.0.0",
+ "officeparser": "7.5.1",
  "slim-select": "2.9.2",
  "stemmer": "2.0.1",
  "tailwind-merge": "3.6.0",

frontend/src/ts/components/modals/CustomTextModal.tsx

@@ -21,6 +21,7 @@ import { getLoadedChallenge, setLoadedChallenge } from "../../states/test";
 import * as CustomText from "../../test/custom-text";
 import * as PractiseWords from "../../test/practise-words";
 import { cn } from "../../utils/cn";
+import { convertToText } from "../../utils/fileparser";
 import * as Strings from "../../utils/strings";
 import { AnimatedModal } from "../common/AnimatedModal";
 import { Button } from "../common/Button";
@@ -320,25 +321,39 @@ export function CustomTextModal(): JSXElement {
  });
  };
 
- const handleFileOpen = () => {
+ const handleFileOpen = async () => {
  const file = fileInputRef?.files?.[0];
  if (!file) return;
 
- if (file.type !== "text/plain") {
- showErrorNotification("File is not a text file", { durationMs: 5000 });
+ const fileExtension = file.name.split(".").pop()?.toLowerCase() ?? "";
+ const isTextFile = file.type === "text/plain" || fileExtension === "txt";
+
+ if (isTextFile) {
+ const reader = new FileReader();
+ reader.readAsText(file, "UTF-8");
+ reader.onload = (e) => {
+ const content = e.target?.result as string;
+ const text = content;
+ form.setFieldValue("text", text);
+ fileInputRef.value = "";
+ };
+ reader.onerror = () => {
+ showErrorNotification("Failed to read file", { durationMs: 5000 });
+ };
+ } else {
+ try {
+ const text = await convertToText(file, file.name, file.type);
+ form.setFieldValue("text", text);
+ fileInputRef.value = "";
+ } catch (error) {
+ const message =
+ error instanceof Error ? error.message : "Failed to parse file";
+ showErrorNotification(`Failed to parse file: ${message}`, {
+ durationMs: 5000,
+ });
+ }
  return;
  }
-
- const reader = new FileReader();
- reader.readAsText(file, "UTF-8");
- reader.onload = (e) => {
- const content = e.target?.result as string;
- form.setFieldValue("text", content);
- fileInputRef.value = "";
- };
- reader.onerror = () => {
- showErrorNotification("Failed to read file", { durationMs: 5000 });
- };
  };
 
  const handleTextareaKeydown = (e: KeyboardEvent) => {
@@ -618,7 +633,7 @@ export function CustomTextModal(): JSXElement {
  ref={fileInputRef}
  type="file"
  class="hidden"
- accept=".txt"
+ accept=".txt,.md,.docx,.doc,.odt,.pdf,.rtf,.epub"
  onChange={handleFileOpen}
  />
  <Button

frontend/src/ts/utils/fileparser.ts

@@ -0,0 +1,55 @@
+let officeparser: typeof import("officeparser") | null = null;
+
+async function loadOfficeparser(): Promise<typeof import("officeparser")> {
+ officeparser ??= await import("officeparser");
+ return officeparser;
+}
+
+type SupportedFileType = "docx" | "odt" | "pdf" | "rtf" | "md" | "epub";
+
+const EXTENSION_MAP: Record<string, SupportedFileType> = {
+ docx: "docx",
+ doc: "docx",
+ odt: "odt",
+ pdf: "pdf",
+ rtf: "rtf",
+ md: "md",
+ epub: "epub",
+};
+
+const MIME_TYPE_MAP: Record<string, SupportedFileType> = {
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document":
+ "docx",
+ "application/vnd.oasis.opendocument.text": "odt",
+ "application/pdf": "pdf",
+ "application/rtf": "rtf",
+ "text/markdown": "md",
+ "text/x-markdown": "md",
+ "application/epub+zip": "epub",
+};
+
+function detectFiletype(
+ filename: string,
+ filetype: string,
+): SupportedFileType | null {
+ const ext = filename.split(".").pop()?.toLowerCase() ?? "";
+ if (EXTENSION_MAP[ext]) return EXTENSION_MAP[ext];
+ if (MIME_TYPE_MAP[filetype]) return MIME_TYPE_MAP[filetype];
+ return null;
+}
+
+export async function convertToText(
+ file: File,
+ filename: string,
+ filetype: string,
+): Promise<string> {
+ const { parseOffice } = await loadOfficeparser();
+ const buffer = await file.arrayBuffer();
+ const fileType = detectFiletype(filename, filetype);
+ const ast = await parseOffice(buffer, { fileType });
+ const { value } = await ast.to("text", {
+ includeImages: false,
+ textConfig: { preserveLayout: false, renderNotes: false },
+ });
+ return value;
+}

frontend/vite.config.ts

@@ -25,7 +25,6 @@ import { VitePWA } from "vite-plugin-pwa";
 import { sentryVitePlugin } from "@sentry/vite-plugin";
 import { KnownFontName } from "@monkeytype/schemas/fonts";
 import solidPlugin from "vite-plugin-solid";
-import devtools from "solid-devtools/vite";
 import tailwindcss from "@tailwindcss/vite";
 
 function getFontsConfig(): string {
@@ -103,9 +102,6 @@ function getPlugins({
  tailwindcss(),
 
  solidPlugin(),
- devtools({
- autoname: true,
- }),
  ];
 
  const devPlugins: PluginOption[] = [
@@ -267,6 +263,10 @@ function getBuildOptions({
  name: "monkeytype-utils",
  test: /src\/ts\/utils\//,
  },
+ {
+ name: "vendor-officeparser",
+ test: /node_modules\/officeparser/,
+ },
  {
  name: "vendor",
  test: /node_modules\//,
@@ -368,7 +368,7 @@ export default defineConfig(({ mode }): UserConfig => {
  root: "src",
  publicDir: "../static",
  optimizeDeps: {
- exclude: ["@fortawesome/fontawesome-free"],
+ exclude: ["@fortawesome/fontawesome-free", "officeparser"],
  },
  };
 });