[reranker] Dom chunks reranking not working as expected

[synacktraa]

Target Site: https://www.vitalproteins.com/products/collagen-gummies
Query: Next button to load paginated reviews
Expected locator: div > div.bv-qna__sc-10ax81p-2.gSPTpM > ul > li:nth-child(2) > a

Analysis keeps resulting in wrong locators because the reranking step is producing non relevant chunks.

Code to reproduce:

package main

import (
	"bufio"
	"context"
	"fmt"
	"log"
	"os"

	"github.com/playwright-community/playwright-go"
	locatr "github.com/vertexcover-io/locatr/pkg"
	"github.com/vertexcover-io/locatr/pkg/llm"
	"github.com/vertexcover-io/locatr/pkg/mode"
	"github.com/vertexcover-io/locatr/pkg/plugins"
)

func main() {
	os.Setenv("ANTHROPIC_API_KEY", "<ANTHROPIC_API_KEY>")
	os.Setenv("COHERE_API_KEY", "<COHERE_API_KEY>")

	pw, err := playwright.Run()
	if err != nil {
		log.Fatalf("could not start Playwright: %v", err)
	}

	browser, err := pw.Chromium.Launch(
		playwright.BrowserTypeLaunchOptions{Headless: playwright.Bool(false)},
	)

	if err != nil {
		log.Fatalf("could not connect to browser: %v", err)
	}

	browserContext, err := browser.NewContext(
		playwright.BrowserNewContextOptions{BypassCSP: playwright.Bool(true)},
	)
	if err != nil {
		log.Fatalf("could not create browser context: %v", err)
	}

	page, err := browserContext.NewPage()
	if err != nil {
		log.Fatalf("could not create new page: %v", err)
	}

	if _, err := page.Goto("https://www.vitalproteins.com/products/collagen-gummies"); err != nil {
		log.Fatalf("failed to load URL: %v", err)
	}

	plugin, err := plugins.NewPlaywrightPlugin(&page)
	if err != nil {
		log.Fatalf("could not create playwright plugin: %v", err)
	}

	llmClient, err := llm.NewLLMClient(
		llm.WithProvider(llm.Anthropic),
		llm.WithModel("claude-3-7-sonnet-latest"),
		llm.WithAPIKey(os.Getenv("ANTHROPIC_API_KEY")),
	)
	if err != nil {
		log.Fatalf("could not create llm client: %v", err)
	}

	locatr, err := locatr.NewLocatr(
		plugin,
		locatr.WithMode(&mode.VisualAnalysisMode{}),
		locatr.WithLLMClient(
			llmClient,
		),
	)
	if err != nil {
		log.Fatal("failed creating locatr", err)
	}

	ctx := context.Background()

	var getCompletion = func(query string) {
		completion, err := locatr.Locate(ctx, query)
		if err != nil {
			log.Print("failed locating element", err)
		}
		fmt.Println(completion)
		fmt.Println(completion.CalculateCost(3.0, 15.0))
		imageBytes, err := locatr.Highlight(ctx, completion.Locators[0], nil)
		if err != nil {
			log.Printf("failed to highlight element: %v", err)
		}

		// Write the image to a file
		err = os.WriteFile("highlighted.png", imageBytes, 0644)
		if err != nil {
			log.Printf("failed to write image: %v", err)
		}
	}

	var pressEnter = func() {
		reader := bufio.NewReader(os.Stdin)
		fmt.Print("Press Enter to continue...")
		_, _ = reader.ReadString('\n') // reads until newline
	}

	getCompletion("Next button to load paginated reviews")
	pressEnter()
}

[kgritesh]

@synacktraa i saw the DOM -> actually the issue is not locatr because you have to click the review button on this site to get reviews -> which locatr cannot do.