Megatron-DeepSpeed/examples/azureml/deepspeed-BLOOM-AML-SDKv2.yaml at 353db918a3a061552aa541e8d67d9b55a35b2f3d · bigscience-workshop/Megatron-DeepSpeed · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
# Training job submission via AML CLI v2

$schema: https://azuremlschemas.azureedge.net/latest/commandJob.schema.json

command: >-
  python pretrain_gpt.py
  --tensor-model-parallel-size 8
  --pipeline-model-parallel-size 10
  --num-layers 80
  --hidden-size 144400
  --num-attention-heads 96
  --seq-length 2048
  --loss-scale 15
  --save-interval 100
  --max-position-embeddings 2048
  --micro-batch-size 1
  --global-batch-size 64
  --lr 6.0e-5
  --min-lr 6.0e-6
  --lr-decay-style 'cosine'
  --log-interval 1
  --eval-iters 40
  --eval-interval 1000
  --clip-grad 1.0
  --weight-decay 0.1
  --adam-beta1 0.9
  --adam-beta2 0.95
  --init-method-std 0.006
  --bf16
  --train-iters 100000
  --data-impl 'mmap'
  --checkpoint-activations
  --tensorboard-dir '/tmp/outputs/tensorboard'
  --deepspeed
  --deepspeed_config 'ds_config.json'
  --zero-stage 0
  --deepspeed-activation-checkpointing
  --exit-interval 500
  --optimizer adam
  --embed-layernorm
  --sync-tp-duplicated-parameters
  --seed 42
  --position-embedding-type alibi
  --tokenizer-type PretrainedFromHF
  --abort-on-unmet-fused-kernel-constraints
  --aml-data-download-path ${{outputs.blobstore_datadir}}
  --data-path 0.033178301 bigscience-catalogue-lm-data_bloom_bin/ar/ar_text_document 0.011279676 bigscience-catalogue-lm-data_bloom_bin/ca/ca_text_document 0.130708086 bigscience-catalogue-lm-data_bloom_bin/code/code_text_document 0.21911033 bigscience-catalogue-lm-data_bloom_bin/en/en_text_document 0.107035252 bigscience-catalogue-lm-data_bloom_bin/es/es_text_document 0.00156473 bigscience-catalogue-lm-data_bloom_bin/eu/eu_text_document 0.130973455 bigscience-catalogue-lm-data_bloom_bin/fr/fr_text_document 0.010954583 bigscience-catalogue-lm-data_bloom_bin/id/id_text_document 0.000110574 bigscience-catalogue-lm-data_bloom_bin/indic-as/indic-as_text_document 0.005510761 bigscience-catalogue-lm-data_bloom_bin/indic-bn/indic-bn_text_document 0.000403458 bigscience-catalogue-lm-data_bloom_bin/indic-gu/indic-gu_text_document 0.007495064 bigscience-catalogue-lm-data_bloom_bin/indic-hi/indic-hi_text_document 0.000621117 bigscience-catalogue-lm-data_bloom_bin/indic-kn/indic-kn_text_document 0.001036982 bigscience-catalogue-lm-data_bloom_bin/indic-ml/indic-ml_text_document 0.000502873 bigscience-catalogue-lm-data_bloom_bin/indic-mr/indic-mr_text_document 0.000669502 bigscience-catalogue-lm-data_bloom_bin/indic-ne/indic-ne_text_document 0.000360473 bigscience-catalogue-lm-data_bloom_bin/indic-or/indic-or_text_document 0.000510136 bigscience-catalogue-lm-data_bloom_bin/indic-pa/indic-pa_text_document 0.002120798 bigscience-catalogue-lm-data_bloom_bin/indic-ta/indic-ta_text_document 0.00091605 bigscience-catalogue-lm-data_bloom_bin/indic-te/indic-te_text_document 0.001249597 bigscience-catalogue-lm-data_bloom_bin/indic-ur/indic-ur_text_document 0.000316939 bigscience-catalogue-lm-data_bloom_bin/nigercongo-all/nigercongo-all_text_document 0.081644439 bigscience-catalogue-lm-data_bloom_bin/oscar-en/oscar-en_text_document 0.055479024 bigscience-catalogue-lm-data_bloom_bin/oscar-zhs/oscar-zhs_text_document 0.049707326 bigscience-catalogue-lm-data_bloom_bin/pt/pt_text_document 0.024698813 bigscience-catalogue-lm-data_bloom_bin/vi/vi_text_document 0.121322237 bigscience-catalogue-lm-data_bloom_bin/zhs/zhs_text_document 0.000519424 bigscience-catalogue-lm-data_bloom_bin/zht/zht_text_document
  --split '90,5,5'
  --tokenizer-name-or-path 'bigscience-catalogue-data-dev/byte-level-bpe-tokenizer-no-norm-250k-whitespace-and-eos-regex-alpha-v3-dedup-lines-articles'
  --adam-eps 1e-8
  --pad-vocab-size-to 250880
  --exit-duration-in-mins 5990
  --pp-partition-method 'type:transformer|embedding'
code: ../../
experiment_name: LargeModel-DistributedJob
environment: azureml:ACPTEnv@latest
environment_variables:
  NCCL_DEBUG: 'WARN'
  NCCL_DEBUG_SUBSYS: 'WARN'
  CUDA_DEVICE_ORDER: 'PCI_BUS_ID'
  DATASET_MOUNT_CACHE_SIZE: '10GB'
  NCCL_SOCKET_IFNAME: 'eth0'
  NCCL_IB_PCI_RELAXED_ORDERING: '1'
  CUDA_LAUNCH_BLOCKING: '1'
  UCX_TLS: 'tcp'
  UCX_NET_DEVICES: 'eth0'
inputs:
  train_file:
    type: uri_file
    mode: download
    path: ../../train-splits.txt
  vocab_file:
    type: uri_file
    mode: download
    path: https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json
  merge_file:
    type: uri_file
    mode: download
    path: https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt
  valid_file:
    type: uri_file
    mode: download
    path: ../../valid-splits.txt
outputs:
  output:
    type: uri_folder
    mode: rw_mount
    path: azureml://datastores/workspaceblobstore/paths/outputs/checkoint
  blobstore_datadir:
    type: uri_folder
    mode: rw_mount
    path: azureml://datastores/bloomdatastore/paths/bloom-data
compute: azureml:bloom
distribution:
  type: pytorch
  process_count_per_instance: 8
resources:
  instance_count: 10