Image-Convolution/workflow.yaml at master · Apollo-Workflows/Image-Convolution · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
---
name: "IMGREC"
dataIns:
- name: "s3bucket"
  type: "string"
  source: "s3bucket"
- name: "desired_num_splits"
  type: "number"
  source: "desired_num_splits"
- name: "s3prefix"
  type: "string"
  source: "s3prefix"
workflowBody:
- function:
    name: "ir-split"
    type: "ir-split"
    dataIns:
    - name: "s3bucket"
      type: "string"
      source: "IMGREC/s3bucket"
    - name: "desired_num_splits"
      type: "number"
      source: "IMGREC/desired_num_splits"
    - name: "s3prefix"
      type: "string"
      source: "IMGREC/s3prefix"
    dataOuts:
    - name: "batches"
      type: "collection"
    - name: "num_splits"
      type: "number"
    properties:
    - name: "resource"
      value: "arn:aws:lambda:us-east-2:735406098573:function:ir-split"
- parallelFor:
    name: "ParallelFor"
    dataIns:
    - name: "images_s3_keys"
      type: "collection"
      source: "ir-split/batches"
      constraints:
      - name: "distribution"
        value: "BLOCK(1)"
    loopCounter:
      type: "number"
      to: "ir-split/num_splits"
      step: "1"
    loopBody:
    - function:
        name: "preprocess-imgs"
        type: "preprocess-imgs"
        dataIns:
        - name: "images_s3_keys"
          type: "collection"
          source: "ParallelFor/images_s3_keys"
        - name: "s3bucket"
          type: "string"
          source: "IMGREC/s3bucket"
        dataOuts:
        - name: "cropped_images_s3_keys"
          type: "collection"
        - name: "cropped_images_timestamps"
          type: "collection"
        properties:
        - name: "resource"
          value: "arn:aws:lambda:us-east-2:735406098573:function:preprocess-imgs"
    - function:
        name: "ir-convolute-reduce"
        type: "ir-convolute-reduce"
        dataIns:
        - name: "cropped_images_s3_keys"
          type: "collection"
          source: "preprocess-imgs/cropped_images_s3_keys"
        - name: "cropped_images_timestamps"
          type: "collection"
          source: "preprocess-imgs/cropped_images_timestamps"
        - name: "s3bucket"
          type: "string"
          source: "IMGREC/s3bucket"
        dataOuts:
        - name: "all_passed"
          type: "bool"
        - name: "invalid_units_frame_keys"
          type: "collection"
        - name: "invalid_units_timestamps"
          type: "collection"
        - name: "ingested_frame_keys"
          type: "collection"
        - name: "ingested_frame_timestamps"
          type: "collection"
        properties:
        - name: "resource"
          value: "arn:aws:lambda:us-east-2:735406098573:function:ir-convolute-reduce"
    dataOuts:
    - name: "invalid_units_frame_keys_batches"
      type: "collection"
      source: "ir-convolute-reduce/invalid_units_frame_keys"
    - name: "invalid_units_frame_timestamps_batches"
      type: "collection"
      source: "ir-convolute-reduce/invalid_units_timestamps"
    - name: "ingested_frame_keys_batches"
      type: "collection"
      source: "ir-convolute-reduce/ingested_frame_keys"
    - name: "ingested_frame_timestamps_batches"
      type: "collection"
      source: "ir-convolute-reduce/ingested_frame_timestamps"
- function:
    name: "ir-reduce"
    type: "ir-reduce"
    dataIns:
    - name: "invalid_units_frame_keys_batches"
      type: "collection"
      source: "ParallelFor/invalid_units_frame_keys_batches"
    - name: "invalid_units_frame_timestamps_batches"
      type: "collection"
      source: "ParallelFor/invalid_units_frame_timestamps_batches"
    - name: "ingested_frame_keys_batches"
      type: "collection"
      source: "ParallelFor/ingested_frame_keys_batches"
    - name: "ingested_frame_timestamps_batches"
      type: "collection"
      source: "ParallelFor/ingested_frame_timestamps_batches"
    dataOuts:
    - name: "average_deviation"
      type: "number"
    properties:
    - name: "resource"
      value: "arn:aws:lambda:us-east-2:735406098573:function:ir-reduce"
dataOuts:
- name: "average_deviation"
  type: "number"
  source: "ir-reduce/average_deviation"
- name: "num_splits"
  type: "number"
  source: "ir-split/num_splits"