initial

.gitignore

@@ -0,0 +1,4 @@
+db
+*.pyo
+*.pyc
+__pycache__

config.py

@@ -0,0 +1,6 @@
+HTTP_HOST = "192.168.1.10"
+HTTP_PORT = 8182
+DATABASE_DIR = "./db"
+QUIET = False
+WEBDATA_DIR = "./web/dist"
+COLLECTION_NAME = 'default'

main.py

@@ -0,0 +1,151 @@
+import asyncio
+import functools
+import time
+
+import chromadb
+from aiohttp import web
+from chromadb.config import Settings
+from chromadb.utils import embedding_functions
+from chromadb.api.models.Collection import Collection
+import logging
+from pathlib import Path
+from config import DATABASE_DIR, HTTP_HOST, HTTP_PORT, QUIET, WEBDATA_DIR, COLLECTION_NAME
+from rich.logging import RichHandler
+from rich.console import Console
+from concurrent.futures import ThreadPoolExecutor
+import uuid
+
+
+log = logging.getLogger(__name__)
+COLLECTION: Collection = None
+executor = ThreadPoolExecutor(1)
+routes = web.RouteTableDef()
+WEBDATA_DIR = Path(WEBDATA_DIR)
+
+async def async_run(func, /, *args, **kwargs):
+    loop = asyncio.get_running_loop()
+    task = loop.run_in_executor(executor, functools.partial(func, *args, **kwargs))
+    results = await asyncio.gather(task)
+    return results[0]
+
+
+@routes.get('/')
+async def serve_index(request):
+    return web.FileResponse(WEBDATA_DIR / 'index.html')
+
+@routes.get('/favicon.ico')
+async def serve_favicon(request):
+    return web.FileResponse(WEBDATA_DIR / 'favicon.ico')
+
+
+@routes.post('/api/v1/search')
+async def add_items(request):
+    data = await request.json()
+    results = await async_run(
+        COLLECTION.query,
+        query_texts=[data.get('text', '')],
+        n_results=10,
+    )
+    items = [
+        [
+            results['ids'][0][i],
+            results['distances'][0][i],
+            results['documents'][0][i],
+            results['metadatas'][0][i],
+        ] for i in range(len(results['ids'][0]))
+    ]
+    return web.json_response(items)
+
+
+@routes.post('/api/v1/item')
+async def add_items(request):
+    data = await request.json()
+    ids = []
+    metadatas = []
+    documents = []
+    body = data.get('body', '')
+    source = data.get('source')
+    search_sources = source == '#'
+    if search_sources:
+        source = ''
+
+    divisions = body.split('\n\n')
+    for items in divisions:
+        lines = [line for line in items.split('\n') if line.strip()]
+        if len(lines) == 0:
+            continue
+        if search_sources and lines[0].startswith('#'):
+            source = lines[0].lstrip('#').strip()
+            continue
+        title = ''
+        if len(lines) >= 2:
+            title = lines[0]
+            lines = lines[1:]
+        else:
+            pass
+        doc = ' '.join(lines)
+        doc = ' '.join(doc.split()).strip() # this just replaces extra whitespaces with one.
+        if len(doc) == 0:
+            continue
+        ids.append(uuid.uuid4().hex)
+        documents.append(doc)
+        metadatas.append({'type': 'question', 'source': source, 'added': int(time.time()), 'title': title})
+    await async_run(COLLECTION.upsert, ids=ids, metadatas=metadatas, documents=documents)
+    return web.json_response({'count': await async_run(COLLECTION.count)})
+
+
+@routes.delete('/api/v1/item')
+async def delete_item(request: web.BaseRequest):
+    await async_run(COLLECTION.delete, ids=[request.query.get('id')])
+    return
+
+
+@routes.put('/api/v1/item')
+async def edit_item(request: web.BaseRequest):
+    data = await request.json()
+    await async_run(
+        COLLECTION.update,
+        ids=[request.query.get('id')],
+        metadatas=[data.get('metadata', {})],
+        documents=[data.get('document', {})]
+    )
+    return
+
+# @routes.get('/api/v1/sources')
+# async def get_sources(request: web.BaseRequest):
+#     await async_run(COLLECTION.get, where=)
+#     return
+
+
+def run():
+    global COLLECTION
+    console = Console(color_system="standard", quiet=QUIET, width=180)
+    logging.basicConfig(level="NOTSET", format="%(message)s", datefmt="[%X]", handlers=[
+        RichHandler(console=console, enable_link_path=False)
+    ])
+
+    log.info("Loading collection: " + COLLECTION_NAME)
+    sentence_transformer_ef = embedding_functions.InstructorEmbeddingFunction(
+        model_name="hkunlp/instructor-large", device="cuda")
+    client = chromadb.PersistentClient(
+        path=DATABASE_DIR,
+        settings=Settings(
+            anonymized_telemetry=False,
+        )
+    )
+    COLLECTION = client.get_or_create_collection(name=COLLECTION_NAME, embedding_function=sentence_transformer_ef)
+    log.info(f"Using web root: {WEBDATA_DIR.absolute()}")
+    app = web.Application(logger=log)
+    app.add_routes(routes)
+    app.router.add_static('/css', WEBDATA_DIR / 'css')
+    app.router.add_static('/fonts', WEBDATA_DIR / 'fonts')
+    app.router.add_static('/img', WEBDATA_DIR / 'img')
+    app.router.add_static('/js', WEBDATA_DIR / 'js')
+
+    log.info(f'Serving http at http://{HTTP_HOST}:{HTTP_PORT}')
+    web.run_app(app, host=HTTP_HOST, port=HTTP_PORT, access_log=log,
+                access_log_format='%a "%r" %s %b "%{Referer}i" "%{User-Agent}i"', print=False)
+
+
+if __name__ == '__main__':
+    run()

requirements.txt

@@ -0,0 +1,5 @@
+chromadb~=0.4.5
+aiohttp~=3.8.4
+rich~=13.5.2
+InstructorEmbedding~=1.0.1
+sentence-transformers~=2.2.2

web/.gitignore

@@ -0,0 +1,23 @@
+.DS_Store
+node_modules
+/dist
+
+
+# local env files
+.env.local
+.env.*.local
+
+# Log files
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+
+# Editor directories and files
+.idea
+.vscode
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

web/README.md

@@ -0,0 +1,24 @@
+# ipt-matcher
+
+## Project setup
+```
+yarn install
+```
+
+### Compiles and hot-reloads for development
+```
+yarn serve
+```
+
+### Compiles and minifies for production
+```
+yarn build
+```
+
+### Lints and fixes files
+```
+yarn lint
+```
+
+### Customize configuration
+See [Configuration Reference](https://cli.vuejs.org/config/).

web/babel.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  presets: [
+    '@vue/cli-plugin-babel/preset'
+  ]
+}

web/jsconfig.json

@@ -0,0 +1,19 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "module": "esnext",
+    "baseUrl": "./",
+    "moduleResolution": "node",
+    "paths": {
+      "@/*": [
+        "src/*"
+      ]
+    },
+    "lib": [
+      "esnext",
+      "dom",
+      "dom.iterable",
+      "scripthost"
+    ]
+  }
+}

web/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "ipt-matcher",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "serve": "vue-cli-service serve",
+    "build": "vue-cli-service build",
+    "lint": "vue-cli-service lint"
+  },
+  "dependencies": {
+    "core-js": "^3.8.3",
+    "primeflex": "^3.3.1",
+    "primeicons": "^6.0.1",
+    "primevue": "^3.32.0",
+    "vue": "^3.2.13"
+  },
+  "devDependencies": {
+    "@babel/core": "^7.12.16",
+    "@babel/eslint-parser": "^7.12.16",
+    "@vue/cli-plugin-babel": "~5.0.0",
+    "@vue/cli-plugin-eslint": "~5.0.0",
+    "@vue/cli-service": "^5.0.8",
+    "eslint": "^7.32.0",
+    "eslint-plugin-vue": "^8.0.3"
+  },
+  "eslintConfig": {
+    "root": true,
+    "env": {
+      "node": true
+    },
+    "extends": [
+      "plugin:vue/vue3-essential",
+      "eslint:recommended"
+    ],
+    "parserOptions": {
+      "parser": "@babel/eslint-parser"
+    },
+    "rules": {}
+  },
+  "browserslist": [
+    "> 1%",
+    "last 2 versions",
+    "not dead",
+    "not ie 11"
+  ]
+}

web/public/index.html

@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html lang="">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width,initial-scale=1.0">
+    <link rel="icon" href="<%= BASE_URL %>favicon.ico">
+    <title><%= htmlWebpackPlugin.options.title %></title>
+  </head>
+  <body>
+    <noscript>
+      <strong>We're sorry but <%= htmlWebpackPlugin.options.title %> doesn't work properly without JavaScript enabled. Please enable it to continue.</strong>
+    </noscript>
+    <div id="app"></div>
+    <!-- built files will be auto injected -->
+  </body>
+</html>

web/src/App.vue

@@ -0,0 +1,14 @@
+<template>
+    <MainPage/>
+</template>
+
+<script>
+import MainPage from './components/MainPage.vue'
+
+export default {
+  name: 'App',
+  components: {
+    MainPage
+  },
+}
+</script>

web/src/components/MainPage.vue

@@ -0,0 +1,166 @@
+<template>
+  <div class="card-container">
+      <ConfirmDialog></ConfirmDialog>
+      <ConfirmDialog group="editing">
+          <template #message="prop">
+            <div class="flex flex-column">
+
+              <label for="edit-title">Title</label>
+              <InputText id="edit-title" v-model="prop.message.item[3].title"/>
+
+              <label for="edit-source">Title</label>
+              <InputText id="edit-source"  v-model="prop.message.item[3].source"/>
+
+              <label>Paragraph</label>
+              <Textarea v-model="prop.message.item[2]" auto-resize rows="5" style="width: 100%"/>
+
+              <label>ID: <i>{{ prop.message.item[0] }}</i></label>
+            </div>
+
+          </template>
+      </ConfirmDialog>
+  <form @submit="onSubmit">
+    <Card>
+      <template #title>IPT Question Search</template>
+      <template #content>
+          <span class="p-float-label">
+            <Textarea v-model="text_area" auto-resize rows="10" style="width: 100%"/>
+            <label>Paragraph</label>
+          </span>
+      </template>
+      <template #footer>
+          <div class="flex flex-row">
+            <Button :loading="search_loading" class="flex align-items-center justify-content-center m-2" icon="pi pi-search" type="submit" label="Search" />
+            <div class="p-inputgroup flex align-items-center justify-content-center m-2">
+              <span class="p-float-label">
+                  <InputText id="source_field" v-model="source" />
+                  <label for="source_field">Source</label>
+              </span>
+              <Button :disabled="source.length === 0" icon="pi pi-plus" severity="secondary" label="Add" @click="onAdd"/>
+            </div>
+          </div>
+      </template>
+    </Card>
+  </form>
+  <Card>
+    <template #title>Results</template>
+    <template #content>
+
+      <DataView :value="results">
+        <template #list="item">
+          <div class="col-12">
+            <div class="flex flex-column">
+              <div class="flex justify-content-between flex-wrap">
+                <div class="flex align-items-center justify-content-center">
+                  <Tag severity="success" :value="item.data[1].toFixed(4)" class="mr-2"></Tag>
+                  <span>
+                    <b>{{ item.data[3].title }}</b>
+                     {{ item.data[3].source }}
+                  </span>
+                </div>
+                <div class="flex align-items-center justify-content-center">
+                  <Button icon="pi pi-file-edit" severity="primary" @click="edit_item(item.data)" text rounded></Button>
+                  <Button icon="pi pi-trash" severity="danger" @click="confirm_delete(item.data[0])" text rounded></Button>
+                </div>
+              </div>
+              <p class="flex align-items-center"> {{ item.data[2] }}</p>
+            </div>
+
+          </div>
+        </template>
+      </DataView>
+    </template>
+  </Card>
+
+</div>
+</template>
+
+<script setup>
+import Card from "primevue/card"
+import Tag from "primevue/tag"
+import Button from "primevue/button"
+import Textarea from "primevue/textarea"
+import InputText from "primevue/inputtext"
+import DataView from 'primevue/dataview';
+import ConfirmDialog from 'primevue/confirmdialog';
+
+import { useConfirm } from "primevue/useconfirm";
+
+import { ref, reactive } from "vue";
+
+const text_area = ref("");
+const source = ref("");
+const search_loading = ref(false);
+const results = reactive([]);
+const confirm = useConfirm();
+
+const confirm_delete = (item_id) => {
+    confirm.require({
+        message: 'Do you want to delete this record?',
+        header: 'Delete Confirmation',
+        icon: 'pi pi-info-circle',
+        acceptClass: 'p-button-danger',
+        accept: async () => {
+            await fetch(`/api/v1/item?id=${item_id}`, { method: "DELETE"} )
+        },
+    });
+};
+
+const edit_item = (item) => {
+    confirm.require({
+        group: 'editing',
+        item: item,
+        header: `Editing "${item[3].title}"`,
+        acceptClass: 'p-button-primal',
+        acceptLabel: 'Edit',
+        rejectLabel: 'Cancel',
+        accept: async () => {
+            await fetch(`/api/v1/item?id=${item[0]}`, { method: "PUT", headers: {
+              "Content-Type": "application/json",
+            }, body: JSON.stringify({document: item[2], metadata: item[3] })} )
+        },
+    });
+};
+
+async function onSubmit(event) {
+  event.preventDefault()
+  event.stopPropagation()
+  search_loading.value = true
+  results.length = 0
+  const response = await fetch('/api/v1/search', {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    }, body: JSON.stringify({text: text_area.value })
+  })
+  let data = await response.json()
+  data.forEach(a => results.push(a))
+  search_loading.value = false
+}
+
+async function onAdd() {
+  await fetch('/api/v1/item', {
+    method: "POST",
+    headers: {
+      "Content-Type": "application/json",
+    }, body: JSON.stringify({body: text_area.value, source: source.value})
+  })
+  text_area.value = ''
+}
+
+</script>
+
+<style>
+.card-container {
+  display: flex;
+  flex-flow: row wrap;
+  justify-content: center;
+  padding-bottom: 5em;
+}
+.card-container .p-card {
+  width: calc(50vw - 2em);
+  min-width: 600px;
+  margin: 0.5em 0.5em 0.5em 0.5em;
+
+}
+</style>

web/src/main.js

@@ -0,0 +1,14 @@
+import { createApp } from 'vue'
+import PrimeVue from 'primevue/config';
+import "primevue/resources/themes/mira/theme.css";
+import "@/theme.css";
+import 'primeicons/primeicons.css';
+import 'primeflex/primeflex.css';
+import ConfirmationService from 'primevue/confirmationservice';
+
+import App from './App.vue'
+
+const app = createApp(App)
+app.use(PrimeVue, { ripple: true });
+app.use(ConfirmationService);
+app.mount('#app');

web/src/theme.css

@@ -0,0 +1,4 @@
+
+body {
+    font-family: var(--font-family);
+}

web/vue.config.js

@@ -0,0 +1,7 @@
+const { defineConfig } = require('@vue/cli-service')
+module.exports = defineConfig({
+  // transpileDependencies: true,
+  devServer: {
+    proxy: {'/api': { target: "http://192.168.1.10:8182" }},
+  },
+})