Add Support for Self-Deployment & Add Feature for Dataset Visualization and Collection

Dockerfile

@@ -0,0 +1,20 @@
+# Use the official Node.js 14 image.
+FROM node:14
+
+# Set the working directory.
+WORKDIR /app
+
+# Copy package.json and package-lock.json
+COPY package*.json ./
+
+# Install dependencies.
+RUN npm install
+
+# Copy local code to the container image.
+COPY . ./
+
+# Build the application
+RUN npm run build
+
+# Start the application
+CMD [ "npm", "start" ]

kubernetes/deployment.yaml

@@ -0,0 +1,33 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: haven-app
+  labels:
+    app: haven
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: haven
+  template:
+    metadata:
+      labels:
+        app: haven
+    spec:
+      containers:
+      - name: haven-app
+        image: haven:latest
+        ports:
+        - containerPort: 8080
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "256Mi"
+          limits:
+            cpu: "500m"
+            memory: "512Mi"
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+      maxSurge: 1

kubernetes/service.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: haven-service
+spec:
+  selector:
+    app: haven
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 8080
+  type: LoadBalancer

main/src/app/api/inference/route.ts

@@ -26,7 +26,7 @@ async function getResponse(host: string, modelId: string | undefined, validatedB
 
 	console.log("sending request", body, host);
 
-	return fetch(host, {
+	return fetch('haven-service', {
 		method: "POST",
 		headers: {
 			"Content-Type": "application/json",
@@ -84,7 +84,7 @@ export async function POST(request: Request) {
 
 	const baseModel = model?.baseModel || defaultModelLoopup[validatedBody.modelId as keyof typeof defaultModelLoopup];
 	const baseModelValidated = y.string().oneOf(modelsToFinetune).required().validateSync(baseModel);
-	const host = inferenceEndpoints[baseModelValidated];
+
 
 	return retryInference(
 		async () => {

main/src/app/datasets/page.tsx

@@ -6,6 +6,7 @@ import DatasetTable from "./table";
 import {getDatasets} from "~/server/database/dataset";
 
 import type {Dataset} from "@prisma/client";
+import DatasetVisualization from './visualization';
 
 function updatedAtToPrettyString(updatedAt: Date) {
 	const now = new Date();
@@ -69,6 +70,7 @@ export default async function Page() {
 			</Padding>
 			<div className="mt-6 border-b border-gray-800" />
 			<DatasetTable datasets={filtered} />
+{/* TODO: Modify DatasetTable or its usage to include buttons for dataset visualization and download. Implement event handlers for these actions. */}
 		</Sidebar>
 	);
 }

main/src/app/datasets/table.tsx

@@ -50,6 +50,9 @@ export default function DatasetTable({datasets}: {datasets: DatasetTableProps})
 										<th scope="col" className="px-3 py-3.5 text-left text-sm font-semibold">
 											Created
 										</th>
+										<th scope="col" className="px-3 py-3.5 text-left text-sm font-semibold">
+											Actions
+										</th>
 									</tr>
 								</thead>
 								<tbody className="divide-y divide-gray-900">
@@ -61,6 +64,10 @@ export default function DatasetTable({datasets}: {datasets: DatasetTableProps})
 											{/*<td className="whitespace-nowrap px-3 py-4 text-sm">{dataset.description}</td>*/}
 											<td className="whitespace-nowrap px-3 py-4 text-sm">{dataset.rows}</td>
 											<td className="whitespace-nowrap px-3 py-4 text-sm">{dataset.created}</td>
+										<td className="whitespace-nowrap px-3 py-4 text-sm">
+											<Button onClick={() => {/* visualization logic here */}}>Visualize</Button>
+											<Button as="a" href={dataset.downloadUrl} download>Download</Button>
+										</td>
 											{/*
 											<td className="relative whitespace-nowrap py-4 pl-3 pr-4 text-right text-sm font-medium sm:pr-0">
 												<a href="#" className="text-gray-100 underline">

main/src/app/datasets/visualization.tsx

@@ -0,0 +1,48 @@
+import React, { useState, useEffect } from 'react';
+import { Button } from '~/components/form/button';
+import { getDatasetDetails } from '~/server/controller/dataset';
+
+const DatasetVisualization = ({ selectedDatasetId }) => {
+  const [datasetDetails, setDatasetDetails] = useState(null);
+  const [isLoading, setIsLoading] = useState(false);
+  const [error, setError] = useState('');
+
+  useEffect(() => {
+    const fetchDatasetDetails = async () => {
+      setIsLoading(true);
+      setError('');
+      try {
+        const details = await getDatasetDetails(selectedDatasetId);
+        setDatasetDetails(details);
+      } catch (err) {
+        setError('Failed to fetch dataset details');
+      } finally {
+        setIsLoading(false);
+      }
+    };
+
+    if (selectedDatasetId) {
+      fetchDatasetDetails();
+    }
+  }, [selectedDatasetId]);
+
+  if (isLoading) return <p>Loading...</p>;
+  if (error) return <p>Error: {error}</p>;
+  if (!datasetDetails) return <p>No dataset selected</p>;
+
+  return (
+    <div>
+      <h2>{datasetDetails.name}</h2>
+      <p>Rows: {datasetDetails.rows}</p>
+      <p>Created: {datasetDetails.created}</p>
+      <Button as="a" href={datasetDetails.downloadUrl} download>
+        Download Dataset
+      </Button>
+      <Button onClick={() => {/* visualization logic here */}}>
+        Visualize Dataset
+      </Button>
+    </div>
+  );
+};
+
+export default DatasetVisualization;

main/src/server/controller/huggingface.ts

@@ -0,0 +1,45 @@
+import axios from 'axios';
+import { createDataset } from '../database/dataset';
+import { downloadFile } from '../utils/modal';
+
+const HUGGINGFACE_API_BASE_URL = 'https://huggingface.co/api';
+
+export async function searchDatasets(query: string) {
+  const response = await axios.get(`${HUGGINGFACE_API_BASE_URL}/datasets/search`, {
+    params: { search: query },
+  });
+  return response.data;
+}
+
+export async function downloadDataset(datasetId: string, userId: string) {
+  const datasetResponse = await axios.get(`${HUGGINGFACE_API_BASE_URL}/datasets/${datasetId}/download`, {
+    responseType: 'blob',
+  });
+
+  const datasetContent = datasetResponse.data;
+  const fileName = `${datasetId}.zip`;
+
+  const downloadUrl = await downloadFile(datasetContent, fileName);
+
+  // Assuming the function to extract metadata from the dataset file exists
+  const { name, rows } = extractMetadataFromDataset(datasetContent);
+
+  await createDataset(userId, name, downloadUrl, rows);
+}
+
+import JSZip from 'jszip';
+
+function extractMetadataFromDataset(datasetContent: Blob): Promise<{ name: string; rows: number }> {
+  return new Promise((resolve, reject) => {
+    const zip = new JSZip();
+    zip.loadAsync(datasetContent)
+      .then(zip => {
+        // Assuming the dataset is in a file named 'data.csv' inside the zip
+        zip.file('data.csv').async('string').then(content => {
+          const rows = content.split('\n').length - 1; // Subtract 1 for the header row
+          const name = 'Extracted Dataset Name'; // Placeholder for actual logic to extract name
+          resolve({ name, rows });
+        }).catch(reject);
+      }).catch(reject);
+  });
+}

main/src/server/database/dataset.ts

@@ -1,15 +1,18 @@
 import {db} from ".";
 
-export async function createDataset(userId: string, name: string, fileName: string, rows: number) {
+export async function createDataset(userId: string, name: string, fileName: string, rows: number, huggingFaceUrl: string, huggingFaceId: string) {
 	return db.dataset.create({
 		data: {
 			userId,
 			name,
 			fileName,
 			rows,
+			huggingFaceUrl,
+			huggingFaceId,
 		},
 	});
 }
+// Note: The database schema needs to be updated to include 'huggingFaceUrl' and 'huggingFaceId' fields. These fields should be of type string.
 
 export async function getDatasets(userId: string) {
 	return db.dataset.findMany({