diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bda7e6a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,20 @@ +# Use the official Node.js 14 image. +FROM node:14 + +# Set the working directory. +WORKDIR /app + +# Copy package.json and package-lock.json +COPY package*.json ./ + +# Install dependencies. +RUN npm install + +# Copy local code to the container image. +COPY . ./ + +# Build the application +RUN npm run build + +# Start the application +CMD [ "npm", "start" ] diff --git a/kubernetes/deployment.yaml b/kubernetes/deployment.yaml new file mode 100644 index 0000000..b8445ef --- /dev/null +++ b/kubernetes/deployment.yaml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: haven-app + labels: + app: haven +spec: + replicas: 3 + selector: + matchLabels: + app: haven + template: + metadata: + labels: + app: haven + spec: + containers: + - name: haven-app + image: haven:latest + ports: + - containerPort: 8080 + resources: + requests: + cpu: "100m" + memory: "256Mi" + limits: + cpu: "500m" + memory: "512Mi" + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + maxSurge: 1 diff --git a/kubernetes/service.yaml b/kubernetes/service.yaml new file mode 100644 index 0000000..d1d509a --- /dev/null +++ b/kubernetes/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: haven-service +spec: + selector: + app: haven + ports: + - protocol: TCP + port: 80 + targetPort: 8080 + type: LoadBalancer diff --git a/main/src/app/api/inference/route.ts b/main/src/app/api/inference/route.ts index 0761c0a..1b84001 100644 --- a/main/src/app/api/inference/route.ts +++ b/main/src/app/api/inference/route.ts @@ -26,7 +26,7 @@ async function getResponse(host: string, modelId: string | undefined, validatedB console.log("sending request", body, host); - return fetch(host, { + return fetch('haven-service', { method: "POST", headers: { "Content-Type": "application/json", @@ -84,7 +84,7 @@ export async function POST(request: Request) { const baseModel = model?.baseModel || defaultModelLoopup[validatedBody.modelId as keyof typeof defaultModelLoopup]; const baseModelValidated = y.string().oneOf(modelsToFinetune).required().validateSync(baseModel); - const host = inferenceEndpoints[baseModelValidated]; + return retryInference( async () => { diff --git a/main/src/app/datasets/page.tsx b/main/src/app/datasets/page.tsx index 929f7f0..dd5a453 100644 --- a/main/src/app/datasets/page.tsx +++ b/main/src/app/datasets/page.tsx @@ -6,6 +6,7 @@ import DatasetTable from "./table"; import {getDatasets} from "~/server/database/dataset"; import type {Dataset} from "@prisma/client"; +import DatasetVisualization from './visualization'; function updatedAtToPrettyString(updatedAt: Date) { const now = new Date(); @@ -69,6 +70,7 @@ export default async function Page() {
+{/* TODO: Modify DatasetTable or its usage to include buttons for dataset visualization and download. Implement event handlers for these actions. */} ); } diff --git a/main/src/app/datasets/table.tsx b/main/src/app/datasets/table.tsx index dece84e..70641f8 100644 --- a/main/src/app/datasets/table.tsx +++ b/main/src/app/datasets/table.tsx @@ -50,6 +50,9 @@ export default function DatasetTable({datasets}: {datasets: DatasetTableProps}) Created + + Actions + @@ -61,6 +64,10 @@ export default function DatasetTable({datasets}: {datasets: DatasetTableProps}) {/*{dataset.description}*/} {dataset.rows} {dataset.created} + + + + {/* diff --git a/main/src/app/datasets/visualization.tsx b/main/src/app/datasets/visualization.tsx new file mode 100644 index 0000000..e53b068 --- /dev/null +++ b/main/src/app/datasets/visualization.tsx @@ -0,0 +1,48 @@ +import React, { useState, useEffect } from 'react'; +import { Button } from '~/components/form/button'; +import { getDatasetDetails } from '~/server/controller/dataset'; + +const DatasetVisualization = ({ selectedDatasetId }) => { + const [datasetDetails, setDatasetDetails] = useState(null); + const [isLoading, setIsLoading] = useState(false); + const [error, setError] = useState(''); + + useEffect(() => { + const fetchDatasetDetails = async () => { + setIsLoading(true); + setError(''); + try { + const details = await getDatasetDetails(selectedDatasetId); + setDatasetDetails(details); + } catch (err) { + setError('Failed to fetch dataset details'); + } finally { + setIsLoading(false); + } + }; + + if (selectedDatasetId) { + fetchDatasetDetails(); + } + }, [selectedDatasetId]); + + if (isLoading) return

Loading...

; + if (error) return

Error: {error}

; + if (!datasetDetails) return

No dataset selected

; + + return ( +
+

{datasetDetails.name}

+

Rows: {datasetDetails.rows}

+

Created: {datasetDetails.created}

+ + +
+ ); +}; + +export default DatasetVisualization; diff --git a/main/src/server/controller/huggingface.ts b/main/src/server/controller/huggingface.ts new file mode 100644 index 0000000..11dd65a --- /dev/null +++ b/main/src/server/controller/huggingface.ts @@ -0,0 +1,45 @@ +import axios from 'axios'; +import { createDataset } from '../database/dataset'; +import { downloadFile } from '../utils/modal'; + +const HUGGINGFACE_API_BASE_URL = 'https://huggingface.co/api'; + +export async function searchDatasets(query: string) { + const response = await axios.get(`${HUGGINGFACE_API_BASE_URL}/datasets/search`, { + params: { search: query }, + }); + return response.data; +} + +export async function downloadDataset(datasetId: string, userId: string) { + const datasetResponse = await axios.get(`${HUGGINGFACE_API_BASE_URL}/datasets/${datasetId}/download`, { + responseType: 'blob', + }); + + const datasetContent = datasetResponse.data; + const fileName = `${datasetId}.zip`; + + const downloadUrl = await downloadFile(datasetContent, fileName); + + // Assuming the function to extract metadata from the dataset file exists + const { name, rows } = extractMetadataFromDataset(datasetContent); + + await createDataset(userId, name, downloadUrl, rows); +} + +import JSZip from 'jszip'; + +function extractMetadataFromDataset(datasetContent: Blob): Promise<{ name: string; rows: number }> { + return new Promise((resolve, reject) => { + const zip = new JSZip(); + zip.loadAsync(datasetContent) + .then(zip => { + // Assuming the dataset is in a file named 'data.csv' inside the zip + zip.file('data.csv').async('string').then(content => { + const rows = content.split('\n').length - 1; // Subtract 1 for the header row + const name = 'Extracted Dataset Name'; // Placeholder for actual logic to extract name + resolve({ name, rows }); + }).catch(reject); + }).catch(reject); + }); +} diff --git a/main/src/server/database/dataset.ts b/main/src/server/database/dataset.ts index 5c72e35..f4041e4 100644 --- a/main/src/server/database/dataset.ts +++ b/main/src/server/database/dataset.ts @@ -1,15 +1,18 @@ import {db} from "."; -export async function createDataset(userId: string, name: string, fileName: string, rows: number) { +export async function createDataset(userId: string, name: string, fileName: string, rows: number, huggingFaceUrl: string, huggingFaceId: string) { return db.dataset.create({ data: { userId, name, fileName, rows, + huggingFaceUrl, + huggingFaceId, }, }); } +// Note: The database schema needs to be updated to include 'huggingFaceUrl' and 'huggingFaceId' fields. These fields should be of type string. export async function getDatasets(userId: string) { return db.dataset.findMany({