diff --git a/backend/src/main/java/com/vaessl/app/sync/HomeboxSyncProvider.java b/backend/src/main/java/com/vaessl/app/sync/HomeboxSyncProvider.java index 9c51258..678932e 100644 --- a/backend/src/main/java/com/vaessl/app/sync/HomeboxSyncProvider.java +++ b/backend/src/main/java/com/vaessl/app/sync/HomeboxSyncProvider.java @@ -1,27 +1,57 @@ package com.vaessl.app.sync; -import org.springframework.stereotype.Service; +import java.util.ArrayList; +import java.util.List; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.stereotype.Component; +import com.vaessl.app.homebox.HomeboxItemClient; +import com.vaessl.app.homebox.HomeboxItemClient.HomeboxItemPage; +import com.vaessl.app.shared.ServiceItem; import com.vaessl.app.shared.ServiceType; import com.vaessl.app.vector.EmbeddingService; +import lombok.RequiredArgsConstructor; -@Service +/** + * Pages through the full Homebox catalog and re-indexes it into the vector store. + * Item IDs are collected across all pages before {@link EmbeddingService#deleteStaleVectorEntries} + * is called once at the end, rather than after each page: deleting per page would treat every + * item outside the current page as stale, wiping out entries from pages already synced. + */ +@Component +@RequiredArgsConstructor public class HomeboxSyncProvider implements SyncProvider { private final EmbeddingService embeddingService; - public HomeboxSyncProvider(EmbeddingService embeddingService) { - this.embeddingService = embeddingService; - } - + private final HomeboxItemClient client; + @Override public ServiceType getServiceType() { return ServiceType.HOMEBOX; } @Override - public void syncVectorStore() { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'syncVectorStore'"); - } + public void syncVectorStore(SyncRequest request) { + int batchSize = 100; + int page = 0; + Page current; + Long connectionId = null; + List currentItemIds = new ArrayList<>(); + do { + Pageable pageable = PageRequest.of(page, batchSize); + HomeboxItemPage result = client.hbResponse(request, null, pageable); + current = result.page(); + if (connectionId == null) { + connectionId = result.connectionId(); + } + embeddingService.vectorizeData(current.getContent(), request.serviceType(), + result.connectionId(), currentItemIds); + page++; + } while (page < current.getTotalPages()); + + embeddingService.deleteStaleVectorEntries(connectionId, currentItemIds); + } } diff --git a/backend/src/main/java/com/vaessl/app/sync/SyncController.java b/backend/src/main/java/com/vaessl/app/sync/SyncController.java new file mode 100644 index 0000000..0982516 --- /dev/null +++ b/backend/src/main/java/com/vaessl/app/sync/SyncController.java @@ -0,0 +1,40 @@ +package com.vaessl.app.sync; + +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; +import com.vaessl.app.shared.SessionKeys; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpSession; +import jakarta.validation.Valid; + +@RestController +public class SyncController { + + private final SyncService syncService; + + public SyncController(SyncService syncService) { + this.syncService = syncService; + } + + /** + * Triggers a vector-store sync for the requested service. Returns {@code 401 Unauthorized} + * if there is no active session. + */ + @PostMapping("/sync") + public ResponseEntity syncVectorStore(@Valid @RequestBody SyncRequest request, + HttpServletRequest httpReq) { + + HttpSession session = httpReq.getSession(false); + + if (session == null + || session.getAttribute(SessionKeys.connectionId(request.serviceType())) == null) { + return ResponseEntity.status(HttpStatus.UNAUTHORIZED).build(); + } + syncService.syncServiceVectorStore(request); + + return ResponseEntity.noContent().build(); + } +} diff --git a/backend/src/main/java/com/vaessl/app/sync/SyncProvider.java b/backend/src/main/java/com/vaessl/app/sync/SyncProvider.java index 03f2b87..846cc48 100644 --- a/backend/src/main/java/com/vaessl/app/sync/SyncProvider.java +++ b/backend/src/main/java/com/vaessl/app/sync/SyncProvider.java @@ -2,6 +2,16 @@ package com.vaessl.app.sync; import com.vaessl.app.shared.ServiceProvider; -public interface SyncProvider extends ServiceProvider{ - public void syncVectorStore(); +/** + * Implemented by any service that supports indexing its items into the vector store. + */ +public interface SyncProvider extends ServiceProvider { + + /** + * Fetches all items from the remote service and indexes them into the vector store, + * removing any previously indexed items that no longer exist upstream. + * + * @param request the sync request containing the app URL and user credentials + */ + public void syncVectorStore(SyncRequest request); } diff --git a/backend/src/main/java/com/vaessl/app/sync/SyncRequest.java b/backend/src/main/java/com/vaessl/app/sync/SyncRequest.java new file mode 100644 index 0000000..5cabd3c --- /dev/null +++ b/backend/src/main/java/com/vaessl/app/sync/SyncRequest.java @@ -0,0 +1,10 @@ +package com.vaessl.app.sync; + +import com.vaessl.app.connection.ConnectionIdentifiable; +import com.vaessl.app.shared.ServiceType; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; + +public record SyncRequest(@NotBlank String appUrl, @NotBlank String username, + @NotNull ServiceType serviceType) implements ConnectionIdentifiable { +} diff --git a/backend/src/main/java/com/vaessl/app/sync/SyncService.java b/backend/src/main/java/com/vaessl/app/sync/SyncService.java new file mode 100644 index 0000000..b2750e9 --- /dev/null +++ b/backend/src/main/java/com/vaessl/app/sync/SyncService.java @@ -0,0 +1,40 @@ +package com.vaessl.app.sync; + +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import org.springframework.stereotype.Service; +import com.vaessl.app.exception.WrongServiceTypeException; +import com.vaessl.app.shared.ServiceType; + +@Service +public class SyncService { + + private final Map providerRegistry; + + public SyncService(List providers) { + Map registry = new EnumMap<>(ServiceType.class); + for (SyncProvider provider : providers) { + registry.put(provider.getServiceType(), provider); + } + this.providerRegistry = registry; + } + + /** + * Dispatches the vector-store sync request to the provider registered for + * {@link SyncRequest#serviceType()}. + * + * @param request the sync request + * @throws WrongServiceTypeException if no provider is registered for the given service type + */ + public void syncServiceVectorStore(SyncRequest request) { + + SyncProvider provider = providerRegistry.get(request.serviceType()); + + if (provider == null) { + throw new WrongServiceTypeException(); + } + + provider.syncVectorStore(request); + } +} diff --git a/backend/src/main/java/com/vaessl/app/vector/EmbeddingService.java b/backend/src/main/java/com/vaessl/app/vector/EmbeddingService.java index 705d4a0..9bd8cfc 100644 --- a/backend/src/main/java/com/vaessl/app/vector/EmbeddingService.java +++ b/backend/src/main/java/com/vaessl/app/vector/EmbeddingService.java @@ -6,10 +6,16 @@ import java.util.List; import java.util.Map; import org.springframework.ai.document.Document; import org.springframework.ai.vectorstore.VectorStore; +import org.springframework.ai.vectorstore.filter.FilterExpressionBuilder; import org.springframework.stereotype.Service; import com.vaessl.app.shared.ServiceItem; import com.vaessl.app.shared.ServiceType; +/** + * Embeds {@link ServiceItem}s into the shared vector store and prunes entries that no longer + * exist upstream. Kept stateless (no instance fields besides {@code vectorStore}) since this is + * a singleton bean shared across concurrent sync runs; callers own the per-sync ID accumulator. + */ @Service public class EmbeddingService { @@ -21,17 +27,31 @@ public class EmbeddingService { this.vectorStore = vectorStore; } - public void vectorizeData(List items, ServiceType serviceType, Long connectionId) { + /** + * Embeds and upserts {@code items} into the vector store, appending each item's ID to + * {@code currentItemIds} so the caller can later prune stale entries via + * {@link #deleteStaleVectorEntries}. Safe to call repeatedly (e.g. once per page) against the + * same accumulator across a single sync run. + * + * @param items the items to embed for this batch + * @param serviceType the originating service, stored in each document's metadata + * @param connectionId the connection these items belong to + * @param currentItemIds accumulator collecting every item ID seen so far in this sync run + */ + public void vectorizeData(List items, ServiceType serviceType, Long connectionId, + List currentItemIds) { List documents = new ArrayList<>(); + for (ServiceItem item : items) { documents.add(toDocument(item, serviceType, connectionId)); + currentItemIds.add(item.id()); } vectorStore.add(documents); } private Document toDocument(ServiceItem item, ServiceType serviceType, Long connectionId) { String extraData = buildExtraData(item.extraData()); - Map metadata = buildMetadata(item, serviceType, connectionId, extraData); + Map metadata = buildMetadata(item, serviceType, connectionId); String content = buildContent(item, extraData); return new Document(connectionId + ":" + item.id(), content, metadata); } @@ -54,30 +74,39 @@ public class EmbeddingService { } private Map buildMetadata(ServiceItem item, ServiceType serviceType, - Long connectionId, String extraData) { + Long connectionId) { Map metadata = new HashMap<>(); metadata.put("connectionId", connectionId); metadata.put("serviceType", serviceType.name()); metadata.put("itemId", item.id()); - metadata.put("title", item.title()); - if (item.description() != null && !item.description().isEmpty()) { - metadata.put("description", item.description()); - } - if (!extraData.isEmpty()) { - metadata.put("extraData", extraData); - } return metadata; } private String buildContent(ServiceItem item, String extraData) { StringBuilder content = new StringBuilder("Title: ").append(item.title()); if (item.description() != null && !item.description().isEmpty()) { - content.append("\n Description: ").append(item.description()); + content.append("\nDescription: ").append(item.description()); } if (!extraData.isEmpty()) { - content.append("\n Extradata: ").append(extraData); + content.append("\nExtradata: ").append(extraData); } return content.toString(); } + /** + * Deletes every vector for {@code connectionId} whose item ID is not in + * {@code currentItemIds}, then clears the accumulator. Must be called once, after every page + * for this sync run has gone through {@link #vectorizeData}, not per page — otherwise items + * from pages other than the most recent one would look stale and get deleted too. + * + * @param connectionId the connection to prune stale vectors for + * @param currentItemIds every item ID seen across the full sync run; cleared after this call + */ + public void deleteStaleVectorEntries(Long connectionId, List currentItemIds) { + FilterExpressionBuilder b = new FilterExpressionBuilder(); + vectorStore.delete(b.and(b.eq("connectionId", connectionId), + b.nin("itemId", new ArrayList<>(currentItemIds))).build()); + + currentItemIds.clear(); + } }