Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions internal/push/parity_golden_test.go
Original file line numberDiff line numberDiff line change
Expand Up@@ -138,6 +138,7 @@ func runGoPreflight(t *testing.T, c parityCase) string {
Root: dir,
LabelsCSV: filepath.Join(dir, c.CSV),
Images: listImages(t, dir),
Sidecars: listSidecars(t, dir),
}
spec := SpecArgs{
Category: c.Category,
Expand DownExpand Up@@ -179,6 +180,41 @@ func listImages(t *testing.T, dir string) []string {
return out
}

// listSidecars maps each sidecar directory in a case (any subdir that isn't
// images/ — annotations/, masks/, texts/, sequences/) to its files, mirroring
// how the production Discover populates LocalLayout.Sidecars. Without this the
// object_detection / semantic_segmentation pairing previews would see an empty
// sidecar set and reject every case (cli#288).
func listSidecars(t *testing.T, dir string) map[string][]string {
t.Helper()
entries, err := os.ReadDir(dir)
if err != nil {
t.Fatal(err)
}
sidecars := make(map[string][]string)
for _, e := range entries {
if !e.IsDir() || e.Name() == "images" {
continue
}
files, err := os.ReadDir(filepath.Join(dir, e.Name()))
if err != nil {
t.Fatal(err)
}
var out []string
for _, f := range files {
if !f.IsDir() {
out = append(out, filepath.Join(dir, e.Name(), f.Name()))
}
}
sort.Strings(out)
sidecars[e.Name()] = out
}
if len(sidecars) == 0 {
return nil
}
return sidecars
}

func mustLoad(t *testing.T, path string, into any) {
t.Helper()
b, err := os.ReadFile(path)
Expand Down
371 changes: 369 additions & 2 deletions internal/push/testdata/parity/cases.json

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/clm-ok/labels.csv
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename
a.txt
b.txt
1 change: 1 addition & 0 deletions internal/push/testdata/parity/cases/clm-ok/texts/a.txt
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
plain pretraining prose with no tabs
1 change: 1 addition & 0 deletions internal/push/testdata/parity/cases/clm-ok/texts/b.txt
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
summarize this article it is about parity
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/emb-malformed/labels.csv
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename
a.txt
b.txt
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
just one field with no tab
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
query text matching passage
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/emb-ok/labels.csv
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename
a.txt
b.txt
1 change: 1 addition & 0 deletions internal/push/testdata/parity/cases/emb-ok/texts/a.txt
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
anchor sentence a close paraphrase
1 change: 1 addition & 0 deletions internal/push/testdata/parity/cases/emb-ok/texts/b.txt
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
query text matching passage unrelated passage
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename,Annotation,Visibility
a.jpg,"not json at all","{""nose"": 1}"
b.jpg,"{""nose"": [6, 7], ""tail"": [18, 22]}","{""nose"": 1, ""tail"": 0}"
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/kp-ok/labels.csv
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename,Annotation,Visibility
a.jpg,"{""nose"": [4, 5], ""tail"": [20, 25]}","{""nose"": 1, ""tail"": 1}"
b.jpg,"{""nose"": [6, 7], ""tail"": [18, 22]}","{""nose"": 1, ""tail"": 0}"
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename,Annotation,Visibility
a.jpg,"{""nose"": [4, 5], ""tail"": [20, 25]}","{""nose"": 1, ""tail"": 1}"
b.jpg,"{""nose"": [6, 7], ""tail"": [18, 22]}","{""nose"": 1, ""tail"": 0}"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename
a.txt
b.txt
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
an unreferenced sequence
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/mlm-ok/labels.csv
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename
a.txt
b.txt
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
the quick brown fox jumps over the lazy dog
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
pack my box with five dozen liquor jugs
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
<annotation>
<folder>images</folder>
<filename>a.jpg</filename>
<segmented>0</segmented>
<object>
<name>widget</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>20</xmin>
<ymin>3</ymin>
<xmax>2</xmax>
<ymax>24</ymax>
</bndbox>
</object>
</annotation>
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
<annotation>
<folder>images</folder>
<filename>b.jpg</filename>
<source>
<database>parity</database>
<annotation>PASCAL VOC</annotation>
</source>
<size>
<width>32</width>
<height>32</height>
<depth>3</depth>
</size>
<segmented>0</segmented>
<object>
<name>widget</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>2</xmin>
<ymin>3</ymin>
<xmax>20</xmax>
<ymax>24</ymax>
</bndbox>
</object>
</annotation>
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename
a.jpg
b.jpg
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
<annotation>
<folder>images</folder>
<filename>a.jpg</filename>
<source>
<database>parity</database>
<annotation>PASCAL VOC</annotation>
</source>
<size>
<width>32</width>
<height>32</height>
<depth>3</depth>
</size>
<segmented>0</segmented>
<object>
<name>widget</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>2</xmin>
<ymin>3</ymin>
<xmax>20</xmax>
<ymax>24</ymax>
</bndbox>
</object>
</annotation>
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename
a.jpg
b.jpg
26 changes: 26 additions & 0 deletions internal/push/testdata/parity/cases/objdet-ok/annotations/a.xml
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
<annotation>
<folder>images</folder>
<filename>a.jpg</filename>
<source>
<database>parity</database>
<annotation>PASCAL VOC</annotation>
</source>
<size>
<width>32</width>
<height>32</height>
<depth>3</depth>
</size>
<segmented>0</segmented>
<object>
<name>widget</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>2</xmin>
<ymin>3</ymin>
<xmax>20</xmax>
<ymax>24</ymax>
</bndbox>
</object>
</annotation>
26 changes: 26 additions & 0 deletions internal/push/testdata/parity/cases/objdet-ok/annotations/b.xml
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
<annotation>
<folder>images</folder>
<filename>b.jpg</filename>
<source>
<database>parity</database>
<annotation>PASCAL VOC</annotation>
</source>
<size>
<width>32</width>
<height>32</height>
<depth>3</depth>
</size>
<segmented>0</segmented>
<object>
<name>widget</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>2</xmin>
<ymin>3</ymin>
<xmax>20</xmax>
<ymax>24</ymax>
</bndbox>
</object>
</annotation>
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/objdet-ok/labels.csv
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename
a.jpg
b.jpg
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
<annotation>
<folder>images</folder>
<filename>a.jpg</filename>
<source>
<database>parity</database>
<annotation>PASCAL VOC</annotation>
</source>
<size>
<width>32</width>
<height>32</height>
<depth>3</depth>
</size>
<segmented>0</segmented>
<object>
<name>widget</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>2</xmin>
<ymin>3</ymin>
<xmax>20</xmax>
<ymax>24</ymax>
</bndbox>
</object>
</annotation>
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
<annotation>
<folder>images</folder>
<filename>b.jpg</filename>
<source>
<database>parity</database>
<annotation>PASCAL VOC</annotation>
</source>
<size>
<width>32</width>
<height>32</height>
<depth>3</depth>
</size>
<segmented>0</segmented>
<object>
<name>widget</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>2</xmin>
<ymin>3</ymin>
<xmax>20</xmax>
<ymax>24</ymax>
</bndbox>
</object>
</annotation>
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
<annotation>
<folder>images</folder>
<filename>c.jpg</filename>
<source>
<database>parity</database>
<annotation>PASCAL VOC</annotation>
</source>
<size>
<width>32</width>
<height>32</height>
<depth>3</depth>
</size>
<segmented>0</segmented>
<object>
<name>widget</name>
<pose>Unspecified</pose>
<truncated>0</truncated>
<difficult>0</difficult>
<bndbox>
<xmin>2</xmin>
<ymin>3</ymin>
<xmax>20</xmax>
<ymax>24</ymax>
</bndbox>
</object>
</annotation>
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename
a.jpg
b.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename,mask_id
a.jpg,a_mask.png
b.jpg,b.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename
a.jpg
b.jpg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename,mask_id
a.jpg,a_mask.png
b.jpg,b_mask.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/semseg-ok/labels.csv
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename,mask_id
a.jpg,a_mask.png
b.jpg,b_mask.png
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename,label
a.txt,entails
b.txt,entails
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
a man is sleeping a person rests
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
water is wet liquid feels wet
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename,label
a.txt,entails
b.txt,contradicts
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
no tab separator in this file
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
the sky is blue the sky is green
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/sentpair-ok/labels.csv
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename,label
a.txt,entails
b.txt,contradicts
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
a man is sleeping a person rests
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
the sky is blue the sky is green
3 changes: 3 additions & 0 deletions internal/push/testdata/parity/cases/seq2seq-ok/labels.csv
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename
a.txt
b.txt
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
hello world hallo welt
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
good morning guten morgen
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
age,income
34,50000
41,62000
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
age,label
34,2.0
41,2.0
29,2.0
4 changes: 4 additions & 0 deletions internal/push/testdata/parity/cases/tabreg-ok/data.csv
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
age,label
34,1.5
41,2.75
29,0.25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename,label
a.txt,pos
b.txt,pos
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
the movie was great
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
loved every minute
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
filename,label
a.txt,PERSON O O
b.txt,B-ORG O B-PER
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
John lives here
Loading
Loading