Merge branch 'main' of https://github.com/ncgr/java

Author: Sam Hokin

chatbot/build.gradle

@@ -53,16 +53,7 @@ dependencies {
     implementation group: 'org.eclipse', name: 'yasson', version: '1.0.11'
 
     // https://mvnrepository.com/artifact/io.pinecone/pinecone-client
-    // implementation group: 'io.pinecone', name: 'pinecone-client', version: '0.2.3'
-
-    // https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java
-    implementation group: 'com.google.protobuf', name: 'protobuf-java', version: '3.23.3'
-    // https://mvnrepository.com/artifact/io.grpc/grpc-protobuf
-    implementation group: 'io.grpc', name: 'grpc-protobuf', version: '1.53.0'
-    implementation group: 'io.grpc', name: 'grpc-stub', version: '1.53.0'
-    implementation group: 'io.grpc', name: 'grpc-netty', version: '1.53.0'
-
-
+    implementation group: 'io.pinecone', name: 'pinecone-client', version: '0.2.3'
 
     // https://mvnrepository.com/artifact/org.projectlombok/lombok
     compileOnly group: 'org.projectlombok', name: 'lombok', version: '1.18.26'

chatbot/libs/ncgr-pubmed.jar

@@ -5,4 +5,4 @@ IDS=$2
 
 curl -X GET \
     -H "Api-Key: $PINECONE_API_KEY" \
-    "https://$INDEX_NAME-index-$PINECONE_PROJECT_NAME.svc.$PINECONE_ENVIRONMENT.pinecone.io/vectors/fetch?ids=$IDS"
+    "https://$INDEX_NAME-$PINECONE_PROJECT_NAME.svc.$PINECONE_ENVIRONMENT.pinecone.io/vectors/fetch?ids=$IDS"

chatbot/libs/pinecone-java-client-0.2.3.jar

@@ -151,12 +151,15 @@ public static void main(String[] args) throws IOException, ParserConfigurationEx
         }
 
         // remove abstracts that lack text
+	int emptyCount = 0;
         List<Abstract> all = new ArrayList<>(abstracts); // avoid concurrent mod
         for (Abstract a : all) {
             if ((a.getText() == null) || (a.getText().length() == 0)) {
                 abstracts.remove(a);
+		emptyCount++;
             }
         }
+	System.out.println("## Removed " + emptyCount + " empty abstracts.");
 
 	// upsert our abstracts
 	if (abstracts!=null && abstracts.size()>0) {

chatbot/scripts/curl-fetch.sh

@@ -135,8 +135,9 @@ public static void main(String[] args) throws IOException, ParserConfigurationEx
                         if (!existingVectors.containsKey(id)) pmidsToUpsert.add(pmid);
                     }
                     if (pmidsToUpsert.size() > 0) {
-                        System.out.println("## Found " + pmidsToUpsert.size() + " new abstracts.");
+                        System.out.println("## " + pmidsToUpsert.size() + " are new PMIDs.");
                         abstracts = Pubmed.getAbstracts(pmidsToUpsert, apikey);
+                        System.out.println("## " + abstracts.size() + " new abstracts were fetched.");
                     }
                 }
             } else {
@@ -162,12 +163,15 @@ public static void main(String[] args) throws IOException, ParserConfigurationEx
         }
 
         // remove abstracts that lack text
+	int emptyCount = 0;
         List<Abstract> all = new ArrayList<>(abstracts); // avoid concurrent mod
         for (Abstract a : all) {
             if ((a.getText() == null) || (a.getText().length() == 0)) {
                 abstracts.remove(a);
+		emptyCount++;
             }
         }
+	System.out.println("## Removed " + emptyCount + " empty abstracts.");
 
 	// upsert the abstracts (only new ones if --update given)
 	if (abstracts!=null && abstracts.size()>0) {

chatbot/src/main/java/org/ncgr/chatbot/PubAgEmbeddingsUpserter.java

@@ -50,5 +50,5 @@ for species in "Aeschynomene evenia" \
                    "Vigna unguiculata"
 do
     echo $species
-    scripts/pubag-embeddings-upserter.sh -i legumebot -u -t "$species"
+    scripts/pubag-embeddings-upserter.sh -i legumebot-index -u -t "$species"
 done

chatbot/src/main/java/org/ncgr/chatbot/PubMedEmbeddingsUpserter.java

@@ -50,5 +50,5 @@ for species in "Aeschynomene evenia" \
                    "Vigna unguiculata"
 do
     echo $species
-    scripts/pubmed-embeddings-upserter.sh -u -i legumebot -r 10000 -t "$species"
+    scripts/pubmed-embeddings-upserter.sh -u -i legumebot-index -r 10000 -t "$species"
 done

chatbot/update-legumebot-pubag.sh

@@ -135,12 +135,12 @@ public List<String> getKeywords() {
     @Override
     public String toString() {
         StringBuffer sb = new StringBuffer();
-        if (title.startsWith("Title: ")) {
+        if (title!=null && title.startsWith("Title: ")) {
             sb.append(title + "\n");
         } else {
             sb.append("Title: " + title + "\n");
         }
-        if (text.startsWith("Abstract: ")) {
+        if (text!=null && text.startsWith("Abstract: ")) {
             sb.append(text + "\n");
         } else {
             sb.append("Abstract: " + text + "\n");

chatbot/update-legumebot-pubmed.sh

@@ -54,6 +54,8 @@
  */
 public class Pubmed {
 
+    final static boolean DEBUG = false;
+
     /**
      * Unmarshal an ESearchResult from a given esearch URI.
      *
@@ -119,13 +121,24 @@ public static PmcArticleset getPmcArticleset(String uri) throws JAXBException, X
      * @return a list of Abstract objects
      */
     static List<Abstract> getAbstracts(PubmedArticleSetDocument articleSetDocument) {
+	int rejectedCount = 0;
         List<Abstract> abstractList = new ArrayList<>();
         for (PubmedArticleType pubmedArticleType : articleSetDocument.getPubmedArticleSet().getPubmedArticleArray()) {
             Abstract a = new Abstract(pubmedArticleType);
             if (a.getTitle()!=null && a.getPMID()!=null) {
                 abstractList.add(a);
-            }
+            } else {
+		rejectedCount++;
+		if (DEBUG) {
+		    System.out.println("## Pubmed.getAbstracts: rejected abstract:");
+		    System.out.println(a);
+		}
+	    }
         }
+	if (DEBUG) {
+	    System.out.println("## Pubmed.getAbstracts: " + rejectedCount + " abstracts lacked title or PMID.");
+	    System.out.println("## Pubmed.getAbstracts: " + abstractList.size() + " abstracts were returned.");
+	}
         return abstractList;
     }