Skip to content

Fix deterministic workspace indexing and search#2178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jkaunert wants to merge 1 commit into
base:main
Choose a base branch
from
Open

Fix deterministic workspace indexing and search #2178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 22 additions & 27 deletions CodeEdit/Features/Documents/WorkspaceDocument/WorkspaceDocument+Find.swift
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -88,7 +88,7 @@ extension WorkspaceDocument.SearchState {
///
/// - Parameter query: The search query to search for.
func search(_ query: String) async {
clearResults()
await resetResults()

await MainActor.run {
self.searchQuery = query
Expand All@@ -104,44 +104,33 @@ extension WorkspaceDocument.SearchState {
}

let asyncController = SearchIndexer.AsyncManager(index: indexer)
let evaluateResultGroup = DispatchGroup()
let evaluateSearchQueue = DispatchQueue(label: "app.codeedit.CodeEdit.EvaluateSearch")

let searchStream = await asyncController.search(query: searchQuery, 20)
for try await result in searchStream {
for file in result.results {
let fileURL = file.url
let fileScore = file.score
let capturedRegexPattern = regexPattern

evaluateSearchQueue.async(group: evaluateResultGroup) {
evaluateResultGroup.enter()
Task { [weak self] in
guard let self else {
evaluateResultGroup.leave()
return
}

let result = await self.evaluateSearchResult(
await withTaskGroup(of: SearchResultModel?.self) { group in
for file in result.results {
let fileURL = file.url
let fileScore = file.score
let capturedRegexPattern = regexPattern

group.addTask { [weak self] in
await self?.evaluateSearchResult(
fileURL: fileURL,
fileScore: fileScore,
regexPattern: capturedRegexPattern
)
}
}

if let result = result {
await self.appendNewResultsToTempResults(newResult: result)
}
evaluateResultGroup.leave()
for await evaluatedResult in group {
if let evaluatedResult {
await appendNewResultsToTempResults(newResult: evaluatedResult)
}
}
}
}

evaluateResultGroup.notify(queue: evaluateSearchQueue) {
Task { @MainActor [weak self] in
self?.setSearchResults()
}
}
await setSearchResults()
}

/// Appends a new search result to the temporary search results array on the main thread.
Expand DownExpand Up@@ -346,7 +335,13 @@ extension WorkspaceDocument.SearchState {

/// Resets the search results along with counts for overall results and file-specific results.
func clearResults() {
DispatchQueue.main.async {
Task {
await resetResults()
}
}

private func resetResults() async {
await MainActor.run {
self.searchResult.removeAll()
self.searchResultsCount = 0
self.searchResultsFileCount = 0
Expand Down
146 changes: 101 additions & 45 deletions CodeEdit/Features/Documents/WorkspaceDocument/WorkspaceDocument+Index.swift
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -11,64 +11,120 @@ extension WorkspaceDocument.SearchState {
/// Adds the contents of the current workspace URL to the search index.
/// That means that the contents of the workspace will be indexed and searchable.
func addProjectToIndex() {
startProjectIndexing()
}

/// Starts project indexing without blocking the caller.
func startProjectIndexing() {
guard let indexer = indexer else { return }
guard let url = workspace.fileURL else { return }

indexStatus = .indexing(progress: 0.0)
let previousTask = indexingTask
previousTask?.cancel()
indexingTask = Task { [weak self] in
await previousTask?.value
guard !Task.isCancelled else { return }

await self?.indexProject(indexer: indexer, url: url)
}
}

/// Indexes the project and returns after the index has been flushed.
func indexProject() async {
let previousTask = indexingTask
previousTask?.cancel()
await previousTask?.value
indexingTask = nil

guard let indexer = indexer else { return }
guard let url = workspace.fileURL else { return }

await indexProject(indexer: indexer, url: url)
}

private func indexProject(indexer: SearchIndexer, url: URL) async {
let uuidString = UUID().uuidString
await publishIndexingStarted(id: uuidString)

let filePaths = getFileURLs(at: url)
let asyncController = SearchIndexer.AsyncManager(index: indexer)
var lastProgress: Double = 0

for await (file, index) in AsyncFileIterator(fileURLs: filePaths) {
guard !Task.isCancelled else {
await publishIndexingCancelled(id: uuidString)
return
}

_ = await asyncController.addText(files: [file], flushWhenComplete: false)
let progress = Double(index + 1) / Double(filePaths.count)

if progress - lastProgress > 0.005 || index == filePaths.count - 1 {
lastProgress = progress
await publishIndexingProgress(id: uuidString, progress: progress)
}
}

guard !Task.isCancelled else {
await publishIndexingCancelled(id: uuidString)
return
}

asyncController.index.flush()
await publishIndexingFinished(id: uuidString)
}

@MainActor
private func publishIndexingStarted(id: String) {
indexStatus = .indexing(progress: 0.0)
let createInfo: [String: Any] = [
"id": uuidString,
"id": id,
"action": "create",
"title": "Indexing | Processing files",
"message": "Creating an index to enable fast and accurate searches within your codebase.",
"isLoading": true
]
NotificationCenter.default.post(name: .taskNotification, object: nil, userInfo: createInfo)
}

Task.detached {
let filePaths = self.getFileURLs(at: url)

let asyncController = SearchIndexer.AsyncManager(index: indexer)
var lastProgress: Double = 0

for await (file, index) in AsyncFileIterator(fileURLs: filePaths) {
_ = await asyncController.addText(files: [file], flushWhenComplete: false)
let progress = Double(index) / Double(filePaths.count)

// Send only if difference is > 0.5%, to keep updates from sending too frequently
if progress - lastProgress > 0.005 || index == filePaths.count - 1 {
lastProgress = progress
await MainActor.run {
self.indexStatus = .indexing(progress: progress)
}
let updateInfo: [String: Any] = [
"id": uuidString,
"action": "update",
"percentage": progress
]
NotificationCenter.default.post(name: .taskNotification, object: nil, userInfo: updateInfo)
}
}
asyncController.index.flush()
@MainActor
private func publishIndexingProgress(id: String, progress: Double) {
indexStatus = .indexing(progress: progress)
let updateInfo: [String: Any] = [
"id": id,
"action": "update",
"percentage": progress
]
NotificationCenter.default.post(name: .taskNotification, object: nil, userInfo: updateInfo)
}

await MainActor.run {
self.indexStatus = .done
}
let updateInfo: [String: Any] = [
"id": uuidString,
"action": "update",
"title": "Finished indexing",
"isLoading": false
]
NotificationCenter.default.post(name: .taskNotification, object: nil, userInfo: updateInfo)

let deleteInfo = [
"id": uuidString,
"action": "deleteWithDelay",
"delay": 4.0
]
NotificationCenter.default.post(name: .taskNotification, object: nil, userInfo: deleteInfo)
}
@MainActor
private func publishIndexingFinished(id: String) {
indexStatus = .done
let updateInfo: [String: Any] = [
"id": id,
"action": "update",
"title": "Finished indexing",
"isLoading": false
]
NotificationCenter.default.post(name: .taskNotification, object: nil, userInfo: updateInfo)

let deleteInfo: [String: Any] = [
"id": id,
"action": "deleteWithDelay",
"delay": 4.0
]
NotificationCenter.default.post(name: .taskNotification, object: nil, userInfo: deleteInfo)
}

@MainActor
private func publishIndexingCancelled(id: String) {
indexStatus = .done
let deleteInfo: [String: Any] = [
"id": id,
"action": "delete"
]
NotificationCenter.default.post(name: .taskNotification, object: nil, userInfo: deleteInfo)
}
Comment thread
jkaunert marked this conversation as resolved.

/// Retrieves an array of file URLs within the specified directory URL.
Expand Down
5 changes: 5 additions & 0 deletions CodeEdit/Features/Documents/WorkspaceDocument/WorkspaceDocument+SearchState.swift
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -38,6 +38,7 @@ extension WorkspaceDocument {
@Published var shouldFocusSearchField: Bool = false

unowned var workspace: WorkspaceDocument
var indexingTask: Task<Void, Never>?
var tempSearchResults = [SearchResultModel]()
var caseSensitive: Bool = false
var indexer: SearchIndexer?
Expand All@@ -53,6 +54,10 @@ extension WorkspaceDocument {
addProjectToIndex()
}

deinit {
indexingTask?.cancel()
}

/// Represents the compare options to be used for find and replace.
///
/// The `replaceOptions` property is a lazy, computed property that dynamically calculates
Expand Down
26 changes: 5 additions & 21 deletions CodeEditTests/Features/Documents/WorkspaceDocument+SearchState+FindAndReplaceTests.swift
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -9,7 +9,7 @@ import XCTest
@testable import CodeEdit

@MainActor
final class FindAndReplaceTests: XCTestCase { // swiftlint:disable:this type_body_length
final class FindAndReplaceTests: XCTestCase {
Comment thread
jkaunert marked this conversation as resolved.
private var directory: URL!
private var files: [CEWorkspaceFile] = []
private var mockWorkspace: WorkspaceDocument!
Expand DownExpand Up@@ -64,20 +64,11 @@ final class FindAndReplaceTests: XCTestCase { // swiftlint:disable:this type_bod
files[1].parent = folder1File
files[2].parent = folder2File

mockWorkspace.searchState?.addProjectToIndex()
await mockWorkspace.searchState?.indexProject()

// NOTE: This is a temporary solution. In the future, a file watcher should track file updates
// and trigger an index update.
let startTime = Date()
let timeoutInSeconds = 2.0
while searchState.indexStatus != .done {
// Check every 0.1 seconds for index completion
try? await Task.sleep(nanoseconds: 100_000_000) // 0.1 seconds
if Date().timeIntervalSince(startTime) > timeoutInSeconds {
XCTFail("TIMEOUT: Indexing took to long or did not complete.")
return
}
}
XCTAssertEqual(searchState.indexStatus, .done)

// Retrieve indexed documents from the indexer
guard let documentsInIndex = searchState.indexer?.documents() else {
Expand All@@ -99,15 +90,8 @@ final class FindAndReplaceTests: XCTestCase { // swiftlint:disable:this type_bod
// IMPORTANT:
// This is only a temporary solution, in the feature a file watcher would track the file update
// and trigger a index update.
searchState.addProjectToIndex()
let startTime = Date()
while searchState.indexStatus != .done {
try? await Task.sleep(nanoseconds: 100_000_000)
if Date().timeIntervalSince(startTime) > 2.0 {
XCTFail("TIMEOUT: Indexing took to long or did not complete.")
return
}
}
await searchState.indexProject()
XCTAssertEqual(searchState.indexStatus, .done)
}

func testFindAndReplace() async {
Expand Down
13 changes: 2 additions & 11 deletions CodeEditTests/Features/Documents/WorkspaceDocument+SearchState+FindTests.swift
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -60,20 +60,11 @@ final class FindTests: XCTestCase {
files[1].parent = parent1
files[2].parent = parent2

await mockWorkspace.searchState?.addProjectToIndex()
await mockWorkspace.searchState?.indexProject()

// The following code also tests whether the workspace is indexed correctly
// Wait until the index is up to date and flushed
let startTime = Date()
let timeoutInSeconds = 2.0
while searchState.indexStatus != .done {
// Check every 0.1 seconds for index completion
try? await Task.sleep(nanoseconds: 100_000_000) // 0.1 seconds
if Date().timeIntervalSince(startTime) > timeoutInSeconds {
XCTFail("TIMEOUT: Indexing took to long or did not complete.")
return
}
}
XCTAssertEqual(searchState.indexStatus, .done)

// Retrieve indexed documents from the indexer
guard let documentsInIndex = searchState.indexer?.documents() else {
Expand Down
13 changes: 2 additions & 11 deletions CodeEditTests/Features/Documents/WorkspaceDocument+SearchState+IndexTests.swift
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -63,20 +63,11 @@ final class WorkspaceDocumentIndexTests: XCTestCase {
files[1].parent = folder1File
files[2].parent = folder2File

await mockWorkspace.searchState?.addProjectToIndex()
await mockWorkspace.searchState?.indexProject()

// The following code also tests whether the workspace is indexed correctly
// Wait until the index is up to date and flushed
let startTime = Date()
let timeoutInSeconds = 2.0
while searchState.indexStatus != .done {
// Check every 0.1 seconds for index completion
try? await Task.sleep(nanoseconds: 100_000_000) // 0.1 seconds
if Date().timeIntervalSince(startTime) > timeoutInSeconds {
XCTFail("TIMEOUT: Indexing took to long or did not complete.")
return
}
}
XCTAssertEqual(searchState.indexStatus, .done)

// Retrieve indexed documents from the indexer
guard let documentsInIndex = searchState.indexer?.documents() else {
Expand Down