Database upload/download + more configs

config.py

@@ -4,3 +4,5 @@ DATABASE_DIR = "./db"
 QUIET = False
 WEBDATA_DIR = "./web/dist"
 COLLECTION_NAME = 'default'
+MODEL_NAME = "hkunlp/instructor-large"  # or "all-MiniLM-L6-v2"
+TORCH_DEVICE = "cuda"  # or 'cpu'

main.py

@@ -1,5 +1,7 @@
 import asyncio
+import codecs
 import functools
+import io
 import time
 
 import chromadb
@@ -9,11 +11,21 @@ from chromadb.utils import embedding_functions
 from chromadb.api.models.Collection import Collection
 import logging
 from pathlib import Path
-from config import DATABASE_DIR, HTTP_HOST, HTTP_PORT, QUIET, WEBDATA_DIR, COLLECTION_NAME
+from config import (
+    DATABASE_DIR,
+    HTTP_HOST,
+    HTTP_PORT,
+    QUIET,
+    WEBDATA_DIR,
+    COLLECTION_NAME,
+    MODEL_NAME,
+    TORCH_DEVICE,
+)
 from rich.logging import RichHandler
 from rich.console import Console
 from concurrent.futures import ThreadPoolExecutor
 import uuid
+import csv
 
 
 log = logging.getLogger(__name__)
@@ -33,6 +45,7 @@ async def async_run(func, /, *args, **kwargs):
 async def serve_index(request):
     return web.FileResponse(WEBDATA_DIR / 'index.html')
 
+
 @routes.get('/favicon.ico')
 async def serve_favicon(request):
     return web.FileResponse(WEBDATA_DIR / 'favicon.ico')
@@ -100,6 +113,40 @@ async def delete_item(request: web.BaseRequest):
     return
 
 
+@routes.get('/api/v1/database')
+async def get_database(request: web.BaseRequest):
+    results = await async_run(COLLECTION.get)
+    buffer = io.StringIO()
+    writer = csv.writer(buffer)
+    writer.writerow(['ID', 'Source', 'Title', 'Document'])
+    for i in range(len(results['ids'])):
+        writer.writerow([
+            results["ids"][i], results['metadatas'][i]['source'],
+            results["metadatas"][i]["title"], results['documents'][i]
+        ])
+    return web.Response(text=buffer.getvalue(), status=200, content_type='text/csv',
+                        headers={'Content-Disposition': 'attachment; filename=ipt_questions.csv'})
+
+
+@routes.post('/api/v1/database')
+async def get_database(request: web.BaseRequest):
+    data = await request.post()
+    ids = []
+    documents = []
+    metadatas = []
+    body = codecs.getreader('utf-8')(data['database'].file)
+    reader = csv.reader(body)
+    header = next(reader)
+    if ','.join(header) != 'ID,Source,Title,Document':
+        return web.Response(text='Invalid CSV header', status=400)
+    for row in reader:
+        ids.append(row[0])
+        documents.append(row[3])
+        metadatas.append({"type": "question", "source": row[1], "added": int(time.time()), "title": row[2]})
+    await async_run(COLLECTION.upsert, ids=ids, metadatas=metadatas, documents=documents)
+    return web.json_response({'count': await async_run(COLLECTION.count)})
+
+
 @routes.put('/api/v1/item')
 async def edit_item(request: web.BaseRequest):
     data = await request.json()
@@ -111,11 +158,6 @@ async def edit_item(request: web.BaseRequest):
     )
     return
 
-# @routes.get('/api/v1/sources')
-# async def get_sources(request: web.BaseRequest):
-#     await async_run(COLLECTION.get, where=)
-#     return
-
 
 def run():
     global COLLECTION
@@ -126,7 +168,7 @@ def run():
 
     log.info("Loading collection: " + COLLECTION_NAME)
     sentence_transformer_ef = embedding_functions.InstructorEmbeddingFunction(
-        model_name="hkunlp/instructor-large", device="cuda")
+        model_name=MODEL_NAME, device=TORCH_DEVICE)
     client = chromadb.PersistentClient(
         path=DATABASE_DIR,
         settings=Settings(

web/src/App.vue

@@ -1,14 +1,190 @@
 <template>
-    <MainPage/>
+  <div class="card-container">
+      <ConfirmDialog></ConfirmDialog>
+      <ConfirmDialog group="editing">
+          <template #message="prop">
+            <div class="flex flex-column">
+
+              <label for="edit-title">Title</label>
+              <InputText id="edit-title" v-model="prop.message.item[3].title"/>
+
+              <label for="edit-source">Title</label>
+              <InputText id="edit-source"  v-model="prop.message.item[3].source"/>
+
+              <label>Paragraph</label>
+              <Textarea v-model="prop.message.item[2]" auto-resize rows="5" style="width: 100%"/>
+
+              <label>ID: <i>{{ prop.message.item[0] }}</i></label>
+            </div>
+
+          </template>
+      </ConfirmDialog>
+  <form @submit="onSubmit">
+    <Card>
+      <template #title>IPT Question Search</template>
+      <template #content>
+          <span class="p-float-label">
+            <Textarea v-model="text_area" auto-resize rows="10" style="width: 100%"/>
+            <label>Paragraph</label>
+          </span>
+      </template>
+      <template #footer>
+          <div class="flex flex-row">
+            <Button :loading="search_loading" class="flex align-items-center justify-content-center m-2" icon="pi pi-search" type="submit" label="Search" />
+            <div class="p-inputgroup flex align-items-center justify-content-center m-2">
+              <span class="p-float-label">
+                  <InputText id="source_field" v-model="source" />
+                  <label for="source_field">Source</label>
+              </span>
+              <Button :disabled="source.length === 0" icon="pi pi-plus" severity="secondary" label="Add" @click="onAdd"/>
+            </div>
+          </div>
+      </template>
+    </Card>
+  </form>
+  <Card>
+    <template #title>Results</template>
+    <template #content>
+
+      <DataView :value="results">
+        <template #list="item">
+          <div class="col-12">
+            <div class="flex flex-column">
+              <div class="flex justify-content-between flex-wrap">
+                <div class="flex align-items-center justify-content-center">
+                  <Tag :severity="tag_colour(item.data[1])" :value="item.data[1].toFixed(4)" class="mr-2"></Tag>
+                  <span>
+                    <b>{{ item.data[3].title }}</b>
+                     {{ item.data[3].source }}
+                  </span>
+                </div>
+                <div class="flex align-items-center justify-content-center">
+                  <Button icon="pi pi-file-edit" severity="primary" @click="edit_item(item.data)" text rounded></Button>
+                  <Button icon="pi pi-trash" severity="danger" @click="confirm_delete(item.data[0])" text rounded></Button>
+                </div>
+              </div>
+              <p class="flex align-items-center"> {{ item.data[2] }}</p>
+            </div>
+          </div>
+        </template>
+      </DataView>
+    </template>
+  </Card>
+  <ProgressSpinner v-show="uploading_db"/>
+  <Accordion :activeIndex="-1" style="width: 100%" class="m-3" v-show="!uploading_db">
+    <AccordionTab header="Options">
+      <div class="flex flex-row">
+        <Button label="Download database" icon="pi pi-download" class="mr-2" @click="download_db"/>
+        <FileUpload mode="basic" name="database" url="/api/v1/database" accept="text/csv"
+                    chooseLabel="Upload database" @before-upload="uploading_db = true" @upload="uploading_db = false"/>
+      </div>
+    </AccordionTab>
+  </Accordion>
+</div>
 </template>
 
-<script>
-import MainPage from './components/MainPage.vue'
+<script setup>
+import Card from "primevue/card"
+import Tag from "primevue/tag"
+import Button from "primevue/button"
+import Textarea from "primevue/textarea"
+import InputText from "primevue/inputtext"
+import DataView from 'primevue/dataview';
+import ConfirmDialog from 'primevue/confirmdialog';
+import { useConfirm } from "primevue/useconfirm";
+import AccordionTab from 'primevue/accordiontab';
+import Accordion from 'primevue/accordion';
+import FileUpload from 'primevue/fileupload';
+import ProgressSpinner from 'primevue/progressspinner';
 
-export default {
-  name: 'App',
-  components: {
-    MainPage
-  },
+import { ref, reactive } from "vue";
+
+const text_area = ref("");
+const source = ref("");
+const search_loading = ref(false);
+const results = reactive([]);
+const confirm = useConfirm();
+const uploading_db = ref(false);
+
+
+const confirm_delete = (item_id) => {
+    confirm.require({
+        message: 'Do you want to delete this record?',
+        header: 'Delete Confirmation',
+        icon: 'pi pi-info-circle',
+        acceptClass: 'p-button-danger',
+        accept: async () => {
+            await fetch(`/api/v1/item?id=${item_id}`, { method: "DELETE"} )
+        },
+    });
+};
+const tag_colour = (dist) => {
+  if (dist < 0.30) return 'success';
+  if (dist < 0.35) return 'warning';
+  return 'danger';
 }
+
+const download_db = () => {
+  window.open('/api/v1/database');
+}
+
+
+
+const edit_item = (item) => {
+    confirm.require({
+        group: 'editing',
+        item: item,
+        header: `Editing "${item[3].title}"`,
+        acceptClass: 'p-button-primal',
+        acceptLabel: 'Edit',
+        rejectLabel: 'Cancel',
+        accept: async () => {
+            await fetch(`/api/v1/item?id=${item[0]}`, { method: "PUT", headers: {
+              "Content-Type": "application/json",
+            }, body: JSON.stringify({document: item[2], metadata: item[3] })} )
+        },
+    });
+};
+
+async function onSubmit(event) {
+  event.preventDefault()
+  event.stopPropagation()
+  search_loading.value = true
+  results.length = 0
+  const response = await fetch('/api/v1/search', {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    }, body: JSON.stringify({text: text_area.value })
+  })
+  let data = await response.json()
+  data.forEach(a => results.push(a))
+  search_loading.value = false
+}
+
+async function onAdd() {
+  await fetch('/api/v1/item', {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    }, body: JSON.stringify({body: text_area.value, source: source.value})
+  })
+  text_area.value = ''
+}
+
 </script>
+
+<style>
+.card-container {
+  display: flex;
+  flex-flow: row wrap;
+  justify-content: center;
+  padding-bottom: 5em;
+}
+.card-container .p-card {
+  width: calc(50vw - 2em);
+  min-width: calc(min(600px, 100vw));
+  margin: 0.5em 0.5em 0.5em 0.5em;
+
+}
+</style>

web/src/components/MainPage.vue

@@ -1,171 +0,0 @@
-<template>
-  <div class="card-container">
-      <ConfirmDialog></ConfirmDialog>
-      <ConfirmDialog group="editing">
-          <template #message="prop">
-            <div class="flex flex-column">
-
-              <label for="edit-title">Title</label>
-              <InputText id="edit-title" v-model="prop.message.item[3].title"/>
-
-              <label for="edit-source">Title</label>
-              <InputText id="edit-source"  v-model="prop.message.item[3].source"/>
-
-              <label>Paragraph</label>
-              <Textarea v-model="prop.message.item[2]" auto-resize rows="5" style="width: 100%"/>
-
-              <label>ID: <i>{{ prop.message.item[0] }}</i></label>
-            </div>
-
-          </template>
-      </ConfirmDialog>
-  <form @submit="onSubmit">
-    <Card>
-      <template #title>IPT Question Search</template>
-      <template #content>
-          <span class="p-float-label">
-            <Textarea v-model="text_area" auto-resize rows="10" style="width: 100%"/>
-            <label>Paragraph</label>
-          </span>
-      </template>
-      <template #footer>
-          <div class="flex flex-row">
-            <Button :loading="search_loading" class="flex align-items-center justify-content-center m-2" icon="pi pi-search" type="submit" label="Search" />
-            <div class="p-inputgroup flex align-items-center justify-content-center m-2">
-              <span class="p-float-label">
-                  <InputText id="source_field" v-model="source" />
-                  <label for="source_field">Source</label>
-              </span>
-              <Button :disabled="source.length === 0" icon="pi pi-plus" severity="secondary" label="Add" @click="onAdd"/>
-            </div>
-          </div>
-      </template>
-    </Card>
-  </form>
-  <Card>
-    <template #title>Results</template>
-    <template #content>
-
-      <DataView :value="results">
-        <template #list="item">
-          <div class="col-12">
-            <div class="flex flex-column">
-              <div class="flex justify-content-between flex-wrap">
-                <div class="flex align-items-center justify-content-center">
-                  <Tag :severity="tag_colour(item.data[1])" :value="item.data[1].toFixed(4)" class="mr-2"></Tag>
-                  <span>
-                    <b>{{ item.data[3].title }}</b>
-                     {{ item.data[3].source }}
-                  </span>
-                </div>
-                <div class="flex align-items-center justify-content-center">
-                  <Button icon="pi pi-file-edit" severity="primary" @click="edit_item(item.data)" text rounded></Button>
-                  <Button icon="pi pi-trash" severity="danger" @click="confirm_delete(item.data[0])" text rounded></Button>
-                </div>
-              </div>
-              <p class="flex align-items-center"> {{ item.data[2] }}</p>
-            </div>
-
-          </div>
-        </template>
-      </DataView>
-    </template>
-  </Card>
-
-</div>
-</template>
-
-<script setup>
-import Card from "primevue/card"
-import Tag from "primevue/tag"
-import Button from "primevue/button"
-import Textarea from "primevue/textarea"
-import InputText from "primevue/inputtext"
-import DataView from 'primevue/dataview';
-import ConfirmDialog from 'primevue/confirmdialog';
-
-import { useConfirm } from "primevue/useconfirm";
-
-import { ref, reactive } from "vue";
-
-const text_area = ref("");
-const source = ref("");
-const search_loading = ref(false);
-const results = reactive([]);
-const confirm = useConfirm();
-
-const confirm_delete = (item_id) => {
-    confirm.require({
-        message: 'Do you want to delete this record?',
-        header: 'Delete Confirmation',
-        icon: 'pi pi-info-circle',
-        acceptClass: 'p-button-danger',
-        accept: async () => {
-            await fetch(`/api/v1/item?id=${item_id}`, { method: "DELETE"} )
-        },
-    });
-};
-const tag_colour = (dist) => {
-  if (dist < 0.30) return 'success';
-  if (dist < 0.35) return 'warning';
-  return 'danger';
-}
-
-const edit_item = (item) => {
-    confirm.require({
-        group: 'editing',
-        item: item,
-        header: `Editing "${item[3].title}"`,
-        acceptClass: 'p-button-primal',
-        acceptLabel: 'Edit',
-        rejectLabel: 'Cancel',
-        accept: async () => {
-            await fetch(`/api/v1/item?id=${item[0]}`, { method: "PUT", headers: {
-              "Content-Type": "application/json",
-            }, body: JSON.stringify({document: item[2], metadata: item[3] })} )
-        },
-    });
-};
-
-async function onSubmit(event) {
-  event.preventDefault()
-  event.stopPropagation()
-  search_loading.value = true
-  results.length = 0
-  const response = await fetch('/api/v1/search', {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-    }, body: JSON.stringify({text: text_area.value })
-  })
-  let data = await response.json()
-  data.forEach(a => results.push(a))
-  search_loading.value = false
-}
-
-async function onAdd() {
-  await fetch('/api/v1/item', {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-    }, body: JSON.stringify({body: text_area.value, source: source.value})
-  })
-  text_area.value = ''
-}
-
-</script>
-
-<style>
-.card-container {
-  display: flex;
-  flex-flow: row wrap;
-  justify-content: center;
-  padding-bottom: 5em;
-}
-.card-container .p-card {
-  width: calc(50vw - 2em);
-  min-width: calc(min(600px, 100vw));
-  margin: 0.5em 0.5em 0.5em 0.5em;
-
-}
-</style>