excluded tag on product, to properly tag "excluded" items (better than

unsetting vertical)

api/src/main/java/org/open4goods/api/controller/api/CompletionController.java

@@ -67,14 +67,14 @@ public CompletionController(VerticalsConfigService verticalsConfigService,
 	@GetMapping("/completion/resources")
 	@Operation(summary = "Launch resource completion on all verticals")
 	public void resourceCompletionAll() throws InvalidParameterException, IOException {
-		resourceCompletionService.completeAll();
+		resourceCompletionService.completeAll(false);
 	}
 
 	@GetMapping("/completion/resources/")
 	@Operation(summary = "Launch resource completion on the specified vertical")
 	public void resourceCompletionVertical(@RequestParam @NotBlank final String verticalConfig)
 			throws InvalidParameterException, IOException {
-		resourceCompletionService.complete(verticalConfigService.getConfigById(verticalConfig));
+		resourceCompletionService.complete(verticalConfigService.getConfigById(verticalConfig),false);
 	}
 
 	@GetMapping("/completion/resources/gtin/")
@@ -97,15 +97,14 @@ public void resourceCompletionProduct(@RequestParam @NotBlank final String gtin)
 	@GetMapping("/completion/genai")
 	@Operation(summary = "Launch genai completion on all verticals")
 	public void genaiCompletionAll() throws InvalidParameterException, IOException {
-		// TODO : From conf
-		aiCompletionService.completeAll(10);
+		aiCompletionService.completeAll(false);
 	}
 
 	@GetMapping("/completion/genai/")
 	@Operation(summary = "Launch genai completion on the specified vertical")
 	public void genaiCompletionVertical(@RequestParam @NotBlank final String verticalConfig, @RequestParam Integer max)
 			throws InvalidParameterException, IOException {
-		aiCompletionService.complete(verticalConfigService.getConfigById(verticalConfig), max);
+		aiCompletionService.complete(verticalConfigService.getConfigById(verticalConfig), max,false);
 	}
 
 	@GetMapping("/completion/genai/gtin/")
@@ -128,14 +127,14 @@ public void genaiCompletionProduct(@RequestParam @NotBlank final String gtin) {
 	@Operation(summary = "Launch amazon completion on all verticals")
 	public void amazonCompletionAll() throws InvalidParameterException, IOException {
 // TODO : From conf
-		amazonCompletionService.completeAll();
+		amazonCompletionService.completeAll(false);
 	}
 
 	@GetMapping("/completion/amazon/")
 	@Operation(summary = "Launch amazon completion on the specified vertical")
 	public void amazonCompletionVertical(@RequestParam @NotBlank final String verticalConfig, @RequestParam Integer max)
 			throws InvalidParameterException, IOException {
-		amazonCompletionService.complete(verticalConfigService.getConfigById(verticalConfig), max);
+		amazonCompletionService.complete(verticalConfigService.getConfigById(verticalConfig), max, false);
 	}
 
 	@GetMapping("/completion/amazon/gtin/")
@@ -162,14 +161,14 @@ public void amazonCompletionProduct(@RequestParam @NotBlank final String gtin) {
 	@Operation(summary = "Launch icecat completion on all verticals")
 	public void icecatCompletionAll() throws InvalidParameterException, IOException {
 // TODO : From conf
-		iceCatService.completeAll();
+		iceCatService.completeAll(true);
 	}
 
 	@GetMapping("/completion/icecat/")
 	@Operation(summary = "Launch icecat completion on the specified vertical")
 	public void icecatCompletionVertical(@RequestParam @NotBlank final String verticalConfig, @RequestParam Integer max)
 			throws InvalidParameterException, IOException {
-		iceCatService.complete(verticalConfigService.getConfigById(verticalConfig), max);
+		iceCatService.complete(verticalConfigService.getConfigById(verticalConfig), max,true);
 	}
 
 	@GetMapping("/completion/icecat/gtin/")

api/src/main/java/org/open4goods/api/services/AbstractCompletionService.java

@@ -30,18 +30,18 @@ public AbstractCompletionService(ProductRepository dataRepository, VerticalsConf
 	/**
 	 * Score verticals with the batch Aggregator
 	 */
-	public void completeAll()  {
-		completeAll(null);
+	public void completeAll(boolean withExcluded)  {
+		completeAll(null,withExcluded);
 	}
 
 	/**
 	 * Score verticals with the batch Aggregator
 	 */
-	public void completeAll(Integer max)  {
-		logger.info("Generating AI texts for all verticals");
+	public void completeAll(Integer max, boolean withExcluded)  {
+		logger.info("Completion for all verticals");
 		for (VerticalConfig vConf : verticalConfigService.getConfigsWithoutDefault()) {
 			if (vConf.getGenAiConfig().isEnabled()) {
-				complete(vConf);				
+				complete(vConf, withExcluded);				
 			}
 		}
 	}
@@ -51,9 +51,9 @@ public void completeAll(Integer max)  {
 	/**
 	 * Proceed to the AI texts generation for a vertical
 	 */
-	public void complete(VerticalConfig vertical, Integer limit)  {
+	public void complete(VerticalConfig vertical, Integer limit, boolean withExcluded)  {
 		logger.info("Generating AI texts for {} products {}",limit == null ? "all" : limit, vertical.getId());
-		dataRepository.exportVerticalWithValidDateOrderByEcoscore(vertical.getId(), limit).forEach(data -> {
+		dataRepository.exportVerticalWithValidDateOrderByEcoscore(vertical.getId(), limit,withExcluded).forEach(data -> {
 			completeProduct(vertical, data);
 
 		});
@@ -68,8 +68,8 @@ public void completeProduct(VerticalConfig vertical, Product data) {
 	/**
 	 * Proceed to the AI texts generation for a vertical
 	 */
-	public void complete(VerticalConfig vertical)  {
-		 this.complete(vertical,null);
+	public void complete(VerticalConfig vertical,boolean withExcluded)  {
+		 this.complete(vertical,null, withExcluded);
 	}
 
 	/**

api/src/main/java/org/open4goods/api/services/AggregationFacadeService.java

@@ -129,7 +129,7 @@ public void score(VerticalConfig vertical)  {
 
 		ScoringBatchedAggregator batchAgg = getScoringAggregator();
 
-		List<Product> productBag = dataRepository.exportVerticalWithValidDate (vertical.getId()).toList();
+		List<Product> productBag = dataRepository.exportVerticalWithValidDate (vertical.getId(), false).toList();
 		// Batched (scoring) aggregation
 		try {
 			batchAgg.score(productBag, verticalConfigService.getConfigByIdOrDefault(vertical.getId()));

api/src/main/java/org/open4goods/api/services/CompletionFacadeService.java

@@ -46,7 +46,7 @@ public CompletionFacadeService(GenAiCompletionService aiCompletionService,
 	@Scheduled(timeUnit = TimeUnit.HOURS, fixedDelay = 12, initialDelay = 1)
 	public void resourceCompletionAll() throws InvalidParameterException, IOException {
 		logger.warn("Completing verticals with resources");
-		resourceCompletionService.completeAll();
+		resourceCompletionService.completeAll(false);
 	}
 
 	///////////////////////////////////
@@ -56,7 +56,7 @@ public void resourceCompletionAll() throws InvalidParameterException, IOExceptio
 	@Scheduled(timeUnit = TimeUnit.HOURS, fixedDelay = 24, initialDelay = 2)
 	public void genaiCompletionAll() throws InvalidParameterException, IOException {
 		logger.warn("Completing verticals with genAI content");
-		aiCompletionService.completeAll();
+		aiCompletionService.completeAll(false);
 	}
 
 	///////////////////////////////////
@@ -65,7 +65,7 @@ public void genaiCompletionAll() throws InvalidParameterException, IOException {
 	@Scheduled(timeUnit = TimeUnit.HOURS, fixedDelay = 24, initialDelay = 3)
 	public void amazonCompletionAll() throws InvalidParameterException, IOException {
 		logger.warn("Completing verticals with amazon");
-		amazonCompletionService.completeAll();
+		amazonCompletionService.completeAll(false);
 	}
 
 	///////////////////////////////////
@@ -74,7 +74,7 @@ public void amazonCompletionAll() throws InvalidParameterException, IOException
 	@Scheduled(timeUnit = TimeUnit.HOURS, fixedDelay = 24, initialDelay = 4)
 	public void icecatCompletionAll() throws InvalidParameterException, IOException {
 		logger.warn("Completing verticals with amazon");
-		icecatCompletionService.completeAll();
+		icecatCompletionService.completeAll(true);
 	}
 
 }

api/src/main/java/org/open4goods/api/services/aggregation/services/realtime/AttributeRealtimeAggregationService.java

@@ -62,7 +62,9 @@ public void onProduct(Product data, VerticalConfig vConf) throws AggregationSkip
 		if (!data.getAttributes().getAggregatedAttributes().keySet().containsAll(vConf.getAttributesConfig().getMandatory())) {
 			// Missing attributes.
 			dedicatedLogger.warn("Missing mandatory attributes for product {}. Will be unmatched from vertical {}", data.getId(), vConf.getId());
-			data.setVertical(null);			
+			data.setExcluded(true);			
+		} else {
+			data.setExcluded(false);			
 		}
 	}
 

commons/src/main/java/org/open4goods/dao/ProductRepository.java

@@ -144,10 +144,14 @@ public Stream<Product> searchInValidPrices(String query, final String indexName,
 	 * @param indexName
 	 * @return
 	 */
-	public Stream<Product> exportVerticalWithValidDate(String vertical) {
+	public Stream<Product> exportVerticalWithValidDate(String vertical, boolean withExcluded) {
 
-		Criteria c = new Criteria("vertical").is(vertical).and(getValidDateQuery());
-		final NativeQuery initialQuery = new NativeQueryBuilder().withQuery(new CriteriaQuery(c)).build();
+		Criteria c = new Criteria("vertical").is(vertical)
+				.and(getValidDateQuery())
+				.and(new Criteria("excluded").is(withExcluded))
+				;
+		final NativeQuery initialQuery = new NativeQueryBuilder()
+				.withQuery(new CriteriaQuery(c)).build();
 		return elasticsearchTemplate.searchForStream(initialQuery, Product.class, current_index).stream()
 				.map(SearchHit::getContent);
 	}
@@ -158,12 +162,16 @@ public Stream<Product> exportVerticalWithValidDate(String vertical) {
 	 * @param vertical
 	 * @param max 
 	 * @param max
+	 * @param withExcluded 
 	 * @param indexName
 	 * @return
 	 */
-	public Stream<Product> exportVerticalWithValidDateOrderByEcoscore(String vertical, Integer max) {
+	public Stream<Product> exportVerticalWithValidDateOrderByEcoscore(String vertical, Integer max, boolean withExcluded) {
 
-		Criteria c = new Criteria("vertical").is(vertical).and(getValidDateQuery());
+		Criteria c = new Criteria("vertical").is(vertical)
+				.and(getValidDateQuery())
+				.and(new Criteria("excluded").is(withExcluded))
+				;
 		NativeQueryBuilder initialQueryBuilder = new NativeQueryBuilder().withQuery(new CriteriaQuery(c)).withMaxResults(max);
 
 				if (null != max) {
@@ -185,8 +193,8 @@ public Stream<Product> exportVerticalWithValidDateOrderByEcoscore(String vertica
 	 * @return
 	 */
 
-	public Stream<Product> exportVerticalWithValidDateOrderByEcoscore(String vertical) {
-		return exportVerticalWithValidDateOrderByEcoscore(vertical, null);
+	public Stream<Product> exportVerticalWithValidDateOrderByEcoscore(String vertical, boolean withExcluded) {
+		return exportVerticalWithValidDateOrderByEcoscore(vertical, null, withExcluded);
 	}
 
 

commons/src/main/java/org/open4goods/helper/DevModeService.java

@@ -107,7 +107,7 @@ public void streamSampleProducts(HttpServletResponse response) throws IOExceptio
 			////////////////////
 			// Export a subset
 			///////////////////
-			repository.exportVerticalWithValidDate(c.getId()).limit(100).forEach(p -> {
+			repository.exportVerticalWithValidDate(c.getId(), false).limit(100).forEach(p -> {
 				try {
 					// Set last offer date to provide a longer product visibility in UI
 					response.getWriter().write(serialisationService.toJson(p)+"\n");

commons/src/main/java/org/open4goods/model/dto/VerticalSearchRequest.java

@@ -34,6 +34,8 @@ public class VerticalSearchRequest {
 
 	Map<String,Set<String>> termsFilter = new HashMap<>();
 
+	// If true, will search in vertical excluded products
+	boolean excluded = false;
 
 	private String sortField;
 	private String sortOrder;
@@ -150,5 +152,14 @@ public void setTermsFilter(Map<String, Set<String>> termsFilter) {
 		this.termsFilter = termsFilter;
 	}
 
+	public boolean isExcluded() {
+		return excluded;
+	}
+
+	public void setExcluded(boolean excluded) {
+		this.excluded = excluded;
+	}
+
+
 
 }

commons/src/main/java/org/open4goods/model/product/Product.java

@@ -94,6 +94,11 @@ public class Product implements Standardisable {
 	@Field(index = true, store = false, type = FieldType.Keyword)
 	private String vertical;
 
+
+	/** If true, means the item is excluded from vertical representation (because not enough data, ....)**/
+	@Field(index = true, store = false, type = FieldType.Boolean)
+	private boolean excluded = false;
+
 	/** The list of other id's known for this product **/
 	// TODO : Rename to alternativ model names
 //	private Set<UnindexedKeyValTimestamp> alternativeIds = new HashSet<>();
@@ -990,6 +995,14 @@ public void setAlternativeModels(Set<String> alternativeModels) {
 		this.alternativeModels = alternativeModels;
 	}
 
+	public boolean isExcluded() {
+		return excluded;
+	}
+
+	public void setExcluded(boolean excluded) {
+		this.excluded = excluded;
+	}
+
 
 
 

commons/src/main/java/org/open4goods/services/SearchService.java

@@ -144,6 +144,7 @@ public VerticalSearchResponse verticalSearch(VerticalConfig vertical, VerticalSe
 
 		Criteria criterias = new Criteria("vertical").is(vertical.getId())
 				.and(aggregatedDataRepository.getValidDateQuery())
+				.and(new Criteria("excluded"). is(request.isExcluded()))
 				;
 
 		// min price

ui/src/main/java/org/open4goods/ui/services/SitemapGenerationService.java

@@ -199,7 +199,7 @@ private void addProductsPages( String baseUrl, String language) {
 		SitemapGenerator sitemap = SitemapGenerator.of(baseUrl);
 
 		for (VerticalConfig vertical : verticalsConfigService.getConfigsWithoutDefault()) {
-			List<Product> datas = aggregatedDataRepository.exportVerticalWithValidDateOrderByEcoscore(vertical.getId())
+			List<Product> datas = aggregatedDataRepository.exportVerticalWithValidDateOrderByEcoscore(vertical.getId(),false)
 			// Filtering on products having genAI content
 					.filter(e -> null != e.getAiDescriptions())
 					// TODO : Not really filtered per language

verticals/src/main/resources/verticals/_default.yml

@@ -39,11 +39,11 @@ i18n:
       ####################################################################################
       # AI TEXTS CONFIGS
       ####################################################################################  
-      aiConfig:
-        - key: "global-description" 
-          prompt: " écris un texte de maximum 100 mots, dans un style académique, une description avec des listes à puce en html, comprenant les principaux avantages et inconvénients de  [(${p.brand()})] [(${p.model()})], sur la base des caracteriques suivantes : [(${p.caracteristics()})]"
-        - key: "ecological-description" 
-          prompt: "écris un texte de maximum 100 mots, dans un style académique, une analyse de l'impact environnemental de [(${p.brand()})] [(${p.model()})], sur la base des caracteriques suivantes : [(${p.caracteristics()})]"
+#      aiConfig:
+#        - key: "global-description" 
+#          prompt: " écris un texte de maximum 100 mots, dans un style académique, une description avec des listes à puce en html, comprenant les principaux avantages et inconvénients de  [(${p.brand()})] [(${p.model()})], sur la base des caracteriques suivantes : [(${p.caracteristics()})]"
+#        - key: "ecological-description" 
+#          prompt: "écris un texte de maximum 100 mots, dans un style académique, une analyse de l'impact environnemental de [(${p.brand()})] [(${p.model()})], sur la base des caracteriques suivantes : [(${p.caracteristics()})]"
 
 
 ####################################################################################