Update job file to include output data type and representative workload from new workload (#140)

Summary:
Pull Request resolved: #140

This diff updates the existing jobs and introduces new jobs for the AI benchmark suite in DCPerf. It specifically adds the output type to the benchmark and includes representative workloads from a new workload.

Reviewed By: excelle08

Differential Revision: D76649961

fbshipit-source-id: d17175f3da56e94786c6ba10c5ad6123e2e33e8a

Author: Kaiwei Tu

benchpress/config/jobs_ai.yml

@@ -1,6 +1,6 @@
 - benchmark: fbgemm_embedding
   name: fbgemm_embedding_a_single
-  description: Embedding workload for ai benchmark with one representative table
+  description: Performance benchmark for Model A workload using one single representative embedding table.
   args:
     - nbit-cpu
     - '--num-embeddings={embeddings}'
@@ -9,62 +9,94 @@
     - '--batch-size={batch_size}'
     - '--num-tables={num_tables}'
     - '--weights-precision={weights_precision}'
+    - '--output-dtype={output_dtype}'
     - '--copies={copies}'
     - '--iters={iters}'
 
   vars:
     - 'embeddings=40000000'
     - 'bag_size=2'
     - 'embedding_dim=96'
-    - 'batch_size=166'
-    - 'num_tables=1'
+    - 'batch_size=162'
+    - 'num_tables=8'
     - 'weights_precision=int4'
+    - 'output_dtype=fp32'
     - 'copies=16'
-    - 'iters=1000'
+    - 'iters=30000'
 
 - benchmark: fbgemm_embedding
   name: fbgemm_embedding_a_spec
-  description: Embedding workload for ai benchmark with 8 representative tables having different bag sizes.
+  description: Performance benchmark for Model A workload using a representative set of embedding tables with different bag sizes.
   args:
     - nbit-device-with-spec
     - '--num-embeddings-list={embeddings_list}'
     - '--bag-size-list={bag_size_list}'
     - '--embedding-dim-list={embedding_dim_list}'
     - '--batch-size={batch_size}'
     - '--weights-precision={weights_precision}'
+    - '--output-dtype={output_dtype}'
     - '--cpu-copies={copies}'
     - '--iters={iters}'
     - '--use-cpu'
 
   vars:
-    - 'embeddings_list=40000000,40000000,40000000,40000000,40000000,40000000,40000000,40000000'
-    - 'bag_size_list=1,1,1,1,1,1,1,5'
-    - 'embedding_dim_list=96,96,96,96,96,96,96,96'
-    - 'batch_size=166'
+    - 'embeddings_list=40000000,40000000,40000000,40000000,40000000,40000000,40000000,40000000,40000000,40000000,40000000,40000000,40000000,40000000,40000000,40000000,40000000'
+    - 'bag_size_list=1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,5,5'
+    - 'embedding_dim_list=96,96,96,96,96,96,96,96,96,96,96,96,96,96,96,96,96'
+    - 'batch_size=168'
     - 'weights_precision=int4'
+    - 'output_dtype=fp32'
     - 'copies=16'
-    - 'iters=1000'
+    - 'iters=20000'
 
 
 - benchmark: fbgemm_embedding
-  name: fbgemm_embedding_b_spec
-  description: Embedding workload for ai benchmark with 8 representative tables having different bag sizes.
+  name: fbgemm_embedding_b_spec_int4
+  description: Performance benchmark for Model B workload using a representative set of embedding tables with int4 precision.
   args:
     - nbit-device-with-spec
     - '--num-embeddings-list={embeddings_list}'
     - '--bag-size-list={bag_size_list}'
     - '--embedding-dim-list={embedding_dim_list}'
     - '--batch-size={batch_size}'
     - '--weights-precision={weights_precision}'
+    - '--output-dtype={output_dtype}'
     - '--cpu-copies={copies}'
     - '--iters={iters}'
     - '--use-cpu'
 
   vars:
-    - 'embeddings_list=100000,100000,100000,100000,100000,5000000,5000000,5000000,10000,10000,14794452'
-    - 'bag_size_list=1,1,1,1,1,1,1,1,1,1,1'
-    - 'embedding_dim_list=64,64,64,64,64,64,64,64,64,64,64'
-    - 'batch_size=166'
+    - 'embeddings_list=5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,14794752,4494000,4494000,4494000,4494000,4494000,4494000,4494000,4494000,4494000,4494000,1697000,1697000,1697000,1697000,1697000,1697000,1697000,1697000,1697000,5000000,5000000,5000000,5000000,5000000,5000000,5000000,2520064,2520064,2520064,2520064,2520064,2520064,2520064,1567000,1567000,1567000,1567000,1567000,1567000,1144000,1144000,1144000,1144000,1144000,1144000,1144000,1144000,1144000,1144000'
+    - 'bag_size_list=1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,5,5,3,3,4'
+    - 'embedding_dim_list=64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64'
+    - 'batch_size=190'
     - 'weights_precision=int4'
+    - 'output_dtype=fp32'
     - 'copies=16'
-    - 'iters=1000'
+    - 'iters=4000'
+
+
+- benchmark: fbgemm_embedding
+  name: fbgemm_embedding_b_spec_int8
+  description: Performance benchmark for Model B workload using a representative set of embedding tables with int8 precision.
+  args:
+    - nbit-device-with-spec
+    - '--num-embeddings-list={embeddings_list}'
+    - '--bag-size-list={bag_size_list}'
+    - '--embedding-dim-list={embedding_dim_list}'
+    - '--batch-size={batch_size}'
+    - '--weights-precision={weights_precision}'
+    - '--output-dtype={output_dtype}'
+    - '--cpu-copies={copies}'
+    - '--iters={iters}'
+    - '--use-cpu'
+
+  vars:
+    - 'embeddings_list=100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,10000,872448,872448,872448,872448,872448,872448,872448,872448,872448,872448,872448,872448,872448,872448,872448,872448,872448,619520,619520,619520,619520,529408,529408,529408,529408,619520,619520,619520,872448,872448,872448,529408,529408,529408,529408,529408,529408,872448,872448,100000,100000,619520,619520,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,619520,100000,100000,100000,100000,100000,619520,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,10000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000,100000'
+    - 'bag_size_list=1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,5,5,5,5,1,1,1,1,2,2,2,2,2,2,2,3,3,3,4,4,4,5,5,5,4,4,1,1,3,3,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,5,1,1,1,1,1,4,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1'
+    - 'embedding_dim_list=64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64,64'
+    - 'batch_size=188'
+    - 'weights_precision=int8'
+    - 'output_dtype=fp32'
+    - 'copies=16'
+    - 'iters=4000'