Vector to abstract dumper/loader; other weaks.

Author: Sam Hokin

chatbot/.gitignore

@@ -0,0 +1 @@
+data/

chatbot/build.gradle

@@ -32,6 +32,8 @@ dependencies {
     implementation group: 'commons-lang', name: 'commons-lang', version: '2.6'
     // https://mvnrepository.com/artifact/commons-cli/commons-cli
     implementation group: 'commons-cli', name: 'commons-cli', version: '1.5.0'
+    // https://mvnrepository.com/artifact/org.apache.commons/commons-text
+    implementation group: 'org.apache.commons', name: 'commons-text', version: '1.10.0'
 
     // https://mvnrepository.com/artifact/org.slf4j/slf4j-simple
     implementation group: 'org.slf4j', name: 'slf4j-simple', version: '2.0.7'
@@ -44,6 +46,9 @@ dependencies {
     // these are for JSON-B serialization
     implementation group: 'javax.json.bind', name: 'javax.json.bind-api', version: '1.0'
 
+    // https://mvnrepository.com/artifact/org.json/json
+    implementation group: 'org.json', name: 'json', version: '20230227'
+
     // https://mvnrepository.com/artifact/org.eclipse/yasson
     implementation group: 'org.eclipse', name: 'yasson', version: '1.0.11'
 

chatbot/scripts/curl-question.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+# usage: DataDumper
+#  -f,--filter          filter the query on metadata given by --key and
+#                       --value
+#  -i,--index <arg>     Pinecone index name
+#  -id,--id <arg>       id of a vector to fetch
+#  -k,--key <arg>       key of the metadata for query filter or update
+#  -t,--term <arg>      search term for query
+#  -topk,--topk <arg>   Pinecone Top K value: maximum number of vectors to
+#                       retrieve [5]
+#  -v,--value <arg>     value of the metadata for query filter or update
+
+java -cp "build/install/ncgr-chatbot/lib/*" org.ncgr.chatbot.DataDumper "$1" "$2" "$3" "$4" "$5" "$6"

chatbot/scripts/datadumper.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+# usage: DataLoader
+#  -f,--file <arg>    Name of file containing vector data and metadata to be
+#                     loaded into Pinecone.
+#  -i,--index <arg>   Pinecone index name
+
+java -cp "build/install/ncgr-chatbot/lib/*" org.ncgr.chatbot.DataLoader $1 $2 $3 $4

chatbot/scripts/dataloader.sh

@@ -0,0 +1,2 @@
+curl "http://localhost:8080/legumebot?question=Which+pathogens+are+most+serious+for+soybean+crops&top_k=5&temperature=0.0&frequency_penalty=0.0&presence_penalty=0.0&show_dois"
+echo ""

chatbot/scripts/jaxrs-question.sh

@@ -1,3 +1 @@
-QUERY=$1
-
-java -cp "build/install/ncgr-chatbot/lib/*" org.ncgr.chatbot.openai.OpenAi "$QUERY"
+java -cp "build/install/ncgr-chatbot/lib/*" org.ncgr.chatbot.openai.OpenAi "$1"

chatbot/scripts/get-openai-embedding.sh chatbot/scripts/openai.shchatbot/scripts/get-openai-embedding.sh renamed to chatbot/scripts/openai.sh

@@ -6,7 +6,4 @@
 #  -p,--page <arg>      page number for search [1]
 #  -t,--term <arg>      search term for abstract and title search
 
-INDEX_NAME=$1
-FILE=$2
-
-java -cp "build/install/ncgr-chatbot/lib/*" org.ncgr.chatbot.PubAgEmbeddingsUpserter -a $PUBAG_API_KEY -i $INDEX_NAME -f $FILE
+java -cp "build/install/ncgr-chatbot/lib/*" org.ncgr.chatbot.PubAgEmbeddingsUpserter -a $PUBAG_API_KEY "$1" "$2" "$3" "$4" "$5" "$6" "$7"

chatbot/scripts/pubag-upsert-file.sh …bot/scripts/pubag-embeddings-upserter.shchatbot/scripts/pubag-upsert-file.sh renamed to chatbot/scripts/pubag-embeddings-upserter.sh chatbot/scripts/pubag-upsert-file.sh renamed to chatbot/scripts/pubag-embeddings-upserter.sh

@@ -1,12 +1,10 @@
 # usage: PubMedEmbeddingsUpserter
 #  -a,--apikey <arg>   PubMed API key
+#  -d,--doi <arg>      DOI for abstract search
 #  -f,--file <arg>     file containing Abstract.toString() data
 #  -i,--index <arg>    Pinecone index name
 #  -l,--list <arg>     comma-separated list of PMIDs
 #  -r,--retmax <arg>   value of retmax for abstract search
 #  -t,--term <arg>     search term for abstract search
 
-INDEX_NAME=$1
-FILE=$2
-
-java -Djavax.xml.accessExternalDTD=https -cp "build/install/ncgr-chatbot/lib/*" org.ncgr.chatbot.PubMedEmbeddingsUpserter -a $PUBMED_API_KEY -i $INDEX_NAME -f $FILE
+java -Djavax.xml.accessExternalDTD=https -cp "build/install/ncgr-chatbot/lib/*" org.ncgr.chatbot.PubMedEmbeddingsUpserter -a $PUBMED_API_KEY "$1" "$2" "$3" "$4" "$5" "$6"