Skip to content

GXflow FullText Search Migration to GitHub #1079

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
LucasLazogue wants to merge 7 commits into
base: master
Choose a base branch
from
Open

GXflow FullText Search Migration to GitHub #1079

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
caad503
GXflow FullText Search Migration to GitHub
LucasLazogue Nov 17, 2025
7923415
centralize dependency versions and remove redundant deps
LucasLazogue Nov 18, 2025
28d56d2
Delete Logger class and use log4j
LucasLazogue Nov 18, 2025
884e5f1
Refactorize code
LucasLazogue Jan 22, 2026
35fd09d
Merge branch 'master' into feat/move-gxflow-full-text-search-to-gxcla…
LucasLazogue Jan 22, 2026
9440967
fix bad merge
LucasLazogue Jan 22, 2026
8103a7c
Trigger Build
LucasLazogue Jan 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions gxflowfulltextsearch/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>com.genexus</groupId>
<artifactId>parent</artifactId>
<version>${revision}${changelist}</version>
</parent>

<artifactId>gxflowfulltextsearch</artifactId>
<name>GXflow FullText Search</name>

<dependencies>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>${commons.collections4.version}</version>
</dependency>
<dependency>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
<version>${commons.logging.version}</version>
</dependency>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
<version>${pdfbox.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>${lucene.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>${poi.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
<version>${log4j.version}</version>
</dependency>
</dependencies>

<build>
<finalName>GXflowFullTextSearch</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.0</version>
<configuration></configuration>
</plugin>
</plugins>
</build>
</project>
25 changes: 25 additions & 0 deletions gxflowfulltextsearch/src/main/java/com/genexus/CA/search/AnalyzerManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package com.genexus.CA.search;

import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;

public class AnalyzerManager {
private static final Map<String, Analyzer> ANALYZERS = new ConcurrentHashMap<>();

static {
ANALYZERS.put("default", new StandardAnalyzer());
// In the future, when the Lucene version is updated, specific analyzers for different languages can be added here.
// For example, for Spanish:
// ANALYZERS.put("es", new org.apache.lucene.analysis.es.SpanishAnalyzer());
}

public static Analyzer getAnalyzer(String lang) {
if (lang == null || lang.trim().isEmpty()) {
return ANALYZERS.get("default");
}
return ANALYZERS.getOrDefault(lang, ANALYZERS.get("default"));
}
}
20 changes: 20 additions & 0 deletions gxflowfulltextsearch/src/main/java/com/genexus/CA/search/IndexManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package com.genexus.CA.search;

import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

public class IndexManager {
private static final Map<String, Indexer> INDEXERS = new ConcurrentHashMap<>();

public static void addContent(String dir, String uri, String lang, String title, String summary, byte fromFile, String body, String filePath) {
getIndexer(dir).addContent(uri, lang, title, summary, fromFile, body, filePath);
}

public static void deleteContent(String dir, String uri) {
getIndexer(dir).deleteContent(uri);
}

private static Indexer getIndexer(String dir) {
return INDEXERS.computeIfAbsent(dir, Indexer::new);
}
}
259 changes: 259 additions & 0 deletions gxflowfulltextsearch/src/main/java/com/genexus/CA/search/Indexer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,259 @@
package com.genexus.CA.search;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.pdfbox.Loader;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.text.PDFTextStripper;

import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;

public final class Indexer {
private String indexDirectory = ".";
private static final int OPERATION_INDEX = 1;
private static final int OPERATION_DELETE = 2;

private static final Logger logger = LogManager.getLogger(Indexer.class);

Indexer(String directory) {
this.indexDirectory = normalizeIndexDirectory(directory);
if (!this.indexExists(this.indexDirectory)) {
try {
IndexWriter writer = new IndexWriter(this.indexDirectory, new StandardAnalyzer(), true);
writer.close();
} catch (Exception e) {
logger.error("Error creating index directory: {}", this.indexDirectory, e);
}
}

}

void addContent(String uri, String lang, String title, String summary, byte fromFile, String body, String filePath) {
Document doc = new Document();
StringBuilder contentBuilder = new StringBuilder();
boolean fileContentRead = false;
String normalizedUri = normalizeUri(uri);
String normalizedLang = normalizeLang(lang);

if (fromFile == 1 && filePath != null && !filePath.trim().isEmpty()) {
String lowerFilePath = filePath.toLowerCase();
try {
if (this.isDocxExtension(lowerFilePath)) {
try (FileInputStream file = new FileInputStream(filePath); XWPFDocument reader = new XWPFDocument(file)) {
for (XWPFParagraph p : reader.getParagraphs()) {
contentBuilder.append(p.getText()).append(" ");
}
fileContentRead = true;
}
} else if (this.isPdfExtension(lowerFilePath)) {
try (PDDocument document = Loader.loadPDF(new File(filePath))) {
PDFTextStripper tStripper = new PDFTextStripper();
contentBuilder.append(tStripper.getText(document));
fileContentRead = true;
}
} else if (this.isTxtExtension(lowerFilePath)) {
contentBuilder.append(readTextFile(filePath));
fileContentRead = true;
}
} catch (IOException e) {
logger.error("Error reading file content from: {}", filePath, e);
}
}

if (body != null && !body.isEmpty() && !fileContentRead) {
contentBuilder.append(body);
}

String content = contentBuilder.toString();

this.indexOperation(OPERATION_DELETE, normalizedLang, null, normalizedUri);

doc.add(new Field("uri", normalizedUri, Store.YES, Index.UN_TOKENIZED));
doc.add(new Field("language", normalizedLang, Store.YES, Index.UN_TOKENIZED));
doc.add(new Field("title", title == null ? "" : title, Store.YES, Index.TOKENIZED));
doc.add(new Field("summary", summary == null ? "" : summary, Store.YES, Index.TOKENIZED));
doc.add(new Field("content", content, Store.YES, Index.TOKENIZED));

try {
this.indexOperation(OPERATION_INDEX, normalizedLang, doc, null);
} catch (Exception e) {
logger.error("Error indexing content. uri={}, lang={}", normalizedUri, normalizedLang, e);
}
}

void deleteContent(String uri) {
try {
this.indexOperation(OPERATION_DELETE, null, null, normalizeUri(uri));
} catch (Exception e) {
logger.error("Error deleting content. uri={}", uri, e);
}

}

private synchronized void indexOperation(int op, String lang, Document doc, String uri) {
switch(op) {
case OPERATION_INDEX:
try {
IndexWriter writer = new IndexWriter(this.getIndexDirectory(), AnalyzerManager.getAnalyzer(lang), false);
writer.addDocument(doc);
// writer.optimize(); // This is a costly operation and should not be done for every document.
writer.close();
} catch (Exception e) {
logger.error("Error indexing document. uri={}, lang={}", uri, lang, e);
}
break;
case OPERATION_DELETE:
IndexReader reader = null;
try {
Term term = null;
int docId = 0;
if (lang == null) {
term = new Term("uri", uri);
} else {
docId = this.getDocumentId(uri, lang);
}

reader = IndexReader.open(this.getIndexDirectory());
if (lang == null) {
reader.deleteDocuments(term);
} else if (docId != -1) {
reader.deleteDocument(docId);
}

} catch (Exception e) {
logger.error("Error deleting document. uri={}, lang={}", uri, lang, e);
} finally {
if (reader != null) {
try {
reader.close();
} catch (IOException e) {
logger.error("Error closing IndexReader", e);
}
}
}
}

}

public String getIndexDirectory() {
return this.indexDirectory;
}

private String normalizeIndexDirectory(String dir) {
if (dir == null || dir.trim().isEmpty()) {
return ".";
}
return new File(dir).getAbsolutePath();
}

private boolean indexExists(String dir) {
try {
new IndexSearcher(dir);
return true;
} catch (IOException e) {
return false;
}
}

private int getDocumentId(String uri, String lang) {
int documentId = -1;

try {
Hits hits = this.getHits(uri, lang);
if (hits.length() > 0) {
documentId = hits.id(0);
}
} catch (IOException e) {
logger.error("Error getting document id. uri={}, lang={}", uri, lang, e);
}

return documentId;
}

private boolean isDocxExtension(String filePath) {
return filePath.toLowerCase().endsWith(".docx");
}

private Hits getHits(String uri, String lang) {
IndexSearcher searcher = null;
Hits hits = null;
try {
searcher = new IndexSearcher(this.indexDirectory);
BooleanQuery query = new BooleanQuery();
query.add(new TermQuery(new Term("uri", uri)), Occur.MUST);
if (lang != null && !lang.trim().isEmpty()) {
query.add(new TermQuery(new Term("language", lang)), Occur.MUST);
}
hits = searcher.search(query);
} catch (IOException e) {
logger.error("Error searching hits. uri={}, lang={}", uri, lang, e);
} finally {
if (searcher != null) {
try {
searcher.close();
} catch (IOException e) {
logger.error("Error closing IndexSearcher", e);
}
}
}

return hits;
}

private String normalizeUri(String uri) {
if (uri == null) {
return "";
}
return uri.trim().toLowerCase();
}

private String normalizeLang(String lang) {
if (lang == null) {
return "";
}
return lang.trim().toLowerCase();
}

private String readTextFile(String filePath) throws IOException {
StringBuilder builder = new StringBuilder();
try (BufferedReader reader = new BufferedReader(new InputStreamReader(Files.newInputStream(Paths.get(filePath)), StandardCharsets.UTF_8))) {
String line;
while ((line = reader.readLine()) != null) {
builder.append(line).append(' ');
}
}
return builder.toString();
}

private boolean isPdfExtension(String filePath) {
return filePath.toLowerCase().endsWith(".pdf");
}

private boolean isTxtExtension(String filePath) {
String lowerFilePath = filePath.toLowerCase();
return lowerFilePath.endsWith(".txt") || lowerFilePath.endsWith(".html");
}
}
Loading
Loading