builder/.github/workflows/gen_data.yaml at bf968cfad60a4694910d7ed3d72a7235eb3e4277 · pkgforge-cargo/builder · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
name: 🦀 Gen Data 🧬
concurrency:
  group: "${{ github.workflow }}-${{ github.ref }}"
  cancel-in-progress: true

on:
  workflow_dispatch:
  schedule:
    #- cron: "0 */2 * * *" #@every 2hrs
    - cron: "30 1 * * *" #7:15 AM NPT
    - cron: "30 13 * * *" #7:15 PM NPT
#------------------------------------------------------------------------------------#
jobs:
    gen-data:
      name: Gen Data
      runs-on: ubuntu-latest
      timeout-minutes: 120
      permissions:
        contents: write

      steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          path: main
          fetch-depth: 1
          filter: "blob:none"

      - name: Setup Env
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          ##Install coreutils
          sudo apt-get update -y -qq && sudo apt-get install curl coreutils dos2unix file findutils gawk git jq moreutils rsync tar xz-utils util-linux wget zip -y -qq
          ##Install Addons
          #https://github.com/pkgforge/devscripts/blob/main/Linux/install_bins_curl.sh
          bash <(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/devscripts/refs/heads/main/Linux/install_bins_curl.sh")
          ##Create Output Dir
          mkdir -p "${GITHUB_WORKSPACE}/main/data"
          echo "GIT_TERMINAL_PROMPT=0" >> "${GITHUB_ENV}"
          echo "GIT_ASKPASS=/bin/echo" >> "${GITHUB_ENV}"
          git config --global "user.email" "AjamX101@gmail.com"
          git config --global "user.name" "Azathothas"
          ##tmp
          SYSTMP="$(dirname $(mktemp -u))" && export SYSTMP="${SYSTMP}"
          echo "SYSTMP=${SYSTMP}" >> "${GITHUB_ENV}"
        continue-on-error: true

      - name: Debloat Runner
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          bash <(curl -qfsSL "https://raw.githubusercontent.com/pkgforge/devscripts/main/Github/Runners/debloat_ubuntu.sh")
        continue-on-error: true

      ## Too Slow & Expensive
      #- name: Generate Metadata (CRATES_DUMP.json)
      #  run: |
      #    #Presets
      #    set +x ; set +e
      #    #--------------#
      #    ##Main
      #    dos2unix --quiet "${GITHUB_WORKSPACE}/main/scripts/_get_initial_list.sh"
      #    chmod +x "${GITHUB_WORKSPACE}/main/scripts/_get_initial_list.sh"
      #    bash "${GITHUB_WORKSPACE}/main/scripts/_get_initial_list.sh"
      #    ##Copy
      #    if [[ -s "${SYSTMP}/CRATES_DUMP.json" && $(stat -c%s "${SYSTMP}/CRATES_DUMP.json") -gt 1024 ]]; then
      #      PKG_COUNT="$(jq -r '.[] | .name' "${SYSTMP}/CRATES_DUMP.json" | grep -iv 'null' | sort -u | wc -l | tr -d '[:space:]')"
      #      if [[ "${PKG_COUNT}" -ge 1000 ]]; then
      #         cp -fv "${SYSTMP}/CRATES_DUMP.json" "${GITHUB_WORKSPACE}/main/data/CRATES_DUMP.json"
      #      fi
      #    fi
      #  continue-on-error: true

      - name: Generate Metadata (CRATES_CMDLINE_ONLY.json)
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          ##Main
          dos2unix --quiet "${GITHUB_WORKSPACE}/main/scripts/_get_process_cmdline_dump.sh"
          chmod +x "${GITHUB_WORKSPACE}/main/scripts/_get_process_cmdline_dump.sh"
          bash "${GITHUB_WORKSPACE}/main/scripts/_get_process_cmdline_dump.sh"
          ##Copy
          if [[ -s "${SYSTMP}/CRATES_CMDLINE_ONLY.json" && $(stat -c%s "${SYSTMP}/CRATES_CMDLINE_ONLY.json") -gt 1024 ]]; then
            PKG_COUNT="$(jq -r '.[] | .name' "${SYSTMP}/CRATES_CMDLINE_ONLY.json" | grep -iv 'null' | sort -u | wc -l | tr -d '[:space:]')"
            if [[ "${PKG_COUNT}" -ge 1000 ]]; then
               cp -fv "${SYSTMP}/CRATES_CMDLINE_ONLY.json" "${GITHUB_WORKSPACE}/main/data/CRATES_CMDLINE_ONLY.json"
            fi
          fi
        continue-on-error: true

      - name: Generate Metadata (CRATES_PROCESSED.json)
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          ##Main
          #dos2unix --quiet "${GITHUB_WORKSPACE}/main/scripts/_process_final_cmdline.sh"
          #chmod +x "${GITHUB_WORKSPACE}/main/scripts/_process_final_cmdline.sh"
          #export USE_GH_LOCAL="YES"
          #bash "${GITHUB_WORKSPACE}/main/scripts/_process_final_cmdline.sh"
          ##Copy
          if [[ -s "${SYSTMP}/CRATES_PROCESSED.json" && $(stat -c%s "${SYSTMP}/CRATES_PROCESSED.json") -gt 1024 ]]; then
            PKG_COUNT="$(jq -r '.[] | .name' "${SYSTMP}/CRATES_PROCESSED.json" | grep -iv 'null' | sort -u | wc -l | tr -d '[:space:]')"
            if [[ "${PKG_COUNT}" -ge 1000 ]]; then
               cp -fv "${SYSTMP}/CRATES_PROCESSED.json" "${GITHUB_WORKSPACE}/main/data/CRATES_PROCESSED.json"
            fi
          fi
        continue-on-error: true

      - name: Get DateTime
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          NEPALI_TIME="$(TZ='Asia/Kathmandu' date +'%Y-%m-%d (%I:%M:%S %p)')"
          echo "NEPALI_TIME=${NEPALI_TIME}" >> "${GITHUB_ENV}"
        continue-on-error: true

      - name: Pull & Push (1)
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          cd "${GITHUB_WORKSPACE}/main"
          git pull origin main --no-edit 2>/dev/null
          git pull origin main --ff-only ; git merge --no-ff -m "Merge & Sync"
        continue-on-error: true
      - uses: stefanzweifel/git-auto-commit-action@v6
        with:
          repository: ./main
          commit_user_name: Azathothas
          commit_user_email: AjamX101@gmail.com
          commit_message: "🦀 Generated Data 🧬 <-- [${{ env.NEPALI_TIME }}] ⌚"

      - name: Pull & Push (2)
        run: |
          #Presets
          set +x ; set +e
          #--------------#
          cd "${GITHUB_WORKSPACE}/main"
          git pull origin main --no-edit 2>/dev/null
          git pull origin main --ff-only ; git merge --no-ff -m "Merge & Sync"
        continue-on-error: true
      - uses: stefanzweifel/git-auto-commit-action@v6
        with:
          repository: ./main
          commit_user_name: Azathothas
          commit_user_email: AjamX101@gmail.com
          commit_message: "🦀 Generated Data 🧬 <-- [${{ env.NEPALI_TIME }}] ⌚"