SoC common : scripts ：Optimize the build check flow and execute time. [2/2]
PD#SWPL-177725
Problem:
Optimize the build check flow and execute time.
Solution:
Optimize the build check flow and execute time.
Verify:
N/A
Change-Id: I89e3ff4009413715ef704f2e7d1a03e26fc8c59e
Signed-off-by: xiaohu.huang <xiaohu.huang@amlogic.com>
diff --git a/build_all.sh b/build_all.sh
index 75174bd..23a030d 100755
--- a/build_all.sh
+++ b/build_all.sh
@@ -5,12 +5,28 @@
# SPDX-License-Identifier: MIT
#
+#Used filter mode and set the repository list with manual mode,plese use '#' to split
+#For example:source scripts/build_all.sh rtos_sdk/arch/arm64#rtos_sdk/libc#
[ -z "$OUTPUT_DIR" ] && OUTPUT_DIR=$PWD/output
[ ! -d $OUTPUT_DIR ] && mkdir -p $OUTPUT_DIR
[ -z "$BUILD_LOG" ] && BUILD_LOG="$OUTPUT_DIR/build.log"
[ -z "$LAST_BUILD_FAILURE" ] && LAST_BUILD_FAILURE="$OUTPUT_DIR/.last_build_failure"
+if [ "$1" == "auto" ]; then
+ CHG_REPO=$(python3 scripts/repositories_changed_filter.py)
+else
+ CHG_REPO="$1"
+fi
+if [ "$CHG_REPO" == "" ]; then
+ use_filter=0
+else
+ echo "Changed repositories:$CHG_REPO"
+ use_filter=1
+fi
+
+failed_numer=0
+
# Clear build.log
cat <<EOF > $BUILD_LOG
EOF
@@ -50,8 +66,22 @@
[ "$?" -ne 0 ] && echo "Ignore unsupported combination! $LINE" && continue
make distclean
[ "$?" -ne 0 ] && echo "Failed to make distclean! $LINE" && return 2
- echo -n -e "$nr. Building $LINE ...\t"
- make >> $BUILD_LOG 2>&1
+ if [ "$use_filter" -eq 1 ]; then
+ make config >> $BUILD_LOG 2>&1
+ array=($LINE)
+ python3 scripts/projects_changed_filter.py "$OUTPUT_DIR" "$LINE" "$CHG_REPO"
+ is_need_compile=$?
+ if [ "$is_need_compile" -eq 1 ]; then
+ echo -n -e "$nr. Building $LINE ...\t"
+ make >> $BUILD_LOG 2>&1
+ else
+ echo -n -e "$nr. $LINE is not changed, building none...\t"
+ :
+ fi
+ else
+ echo -n -e "$nr. Building $LINE ...\t"
+ make >> $BUILD_LOG 2>&1
+ fi
[ "$?" -ne 0 ] && echo "failed!" && cat $BUILD_LOG && touch $LAST_BUILD_FAILURE && echo -e "\nAborted with errors!\n" && return 3
grep -qr "warning: " $BUILD_LOG
[ "$?" -eq 0 ] && echo "with warnings!" && cat $BUILD_LOG && touch $LAST_BUILD_FAILURE && echo -e "\nAborted with warnings!\n" && return 1
diff --git a/projects_changed_filter.py b/projects_changed_filter.py
new file mode 100755
index 0000000..75eac8f
--- /dev/null
+++ b/projects_changed_filter.py
@@ -0,0 +1,126 @@
+#!/usr/bin/python3
+#coding:utf-8
+#
+# Copyright (c) 2021-2022 Amlogic, Inc. All rights reserved.
+#
+# SPDX-License-Identifier: MIT
+#
+
+import sys
+import os
+import argparse
+import subprocess
+
+def is_subpath(path1, path2):
+ common_part = os.path.commonpath([path1, path2])
+ return common_part == path1
+
+class project_c:
+ def __init__(self, full_name):
+ self.full_name = full_name
+ elements = full_name.split(' ')
+ self.arch = elements[0]
+ self.soc = elements[1]
+ self.board = elements[2]
+ self.product = elements[3]
+ self.repository_set = set()
+
+ def bind_repository_set(repository_set):
+ self.repository_set = repository_set
+
+class repository_set_c:
+ def __init__(self):
+ self.container_set = set()
+
+ def add_repository(self, repository_pair):
+ self.container_set.add(repository_pair)
+
+ def get_repository_name(self, bind_dir_path):
+ repository_name = ""
+
+ for repository in self.container_set:
+ if is_subpath(repository[1], bind_dir_path):
+ repository_name = repository[0]
+ return repository_name
+
+ def get_repository_path(self, bind_rep_name):
+ repository_path = ""
+
+ for repository in self.container_set:
+ if bind_rep_name == repository[0]:
+ repository_path = repository[1]
+ return repository_path
+
+def main():
+ parser = argparse.ArgumentParser(description='Filter the build project which is changed.')
+ parser.add_argument('output_path', help="The project output dir path.")
+ parser.add_argument('prj_msg', help="The project description line message")
+ parser.add_argument('changed_reps', help="The changed repository list")
+ args = parser.parse_args()
+ output_path = args.output_path
+ prj_msg = args.prj_msg
+ changed_reps = args.changed_reps
+
+ prj_elements = prj_msg.split(' ')
+ arch = prj_elements[0]
+ soc = prj_elements[1]
+ board = prj_elements[2]
+ product = prj_elements[3]
+ project_path = output_path + "/{}-{}-{}".format(arch, board, product)
+ base_path = os.path.abspath(output_path + "/..")
+
+ project_list_file = output_path + "/build_combination.txt"
+ project_list_file = os.path.abspath(project_list_file)
+ with open(project_list_file) as f:
+ lines = iter(f)
+ for line in lines:
+ project = project_c(line)
+
+ repository_list_file = project_path + "/freertos/rtos_sdk_manifest.xml"
+ all_repository_set = repository_set_c()
+ with open(repository_list_file) as f:
+ lines = iter(f)
+ for line in lines:
+ line = line.strip()
+ if line.find("project name=") < 0:
+ continue
+ else:
+ elements = line.split(' ')
+ name_str = elements[1].split('=')[1][1:-1]
+ path_str = elements[2].split('=')[1][1:-3]
+ repository = (name_str, path_str)
+ all_repository_set.add_repository(repository)
+
+ dirs_file = project_path + "/freertos/module_dirs.txt"
+ prj_repository_set = repository_set_c()
+ prj_repository_unique_set = set()
+ with open(dirs_file) as f:
+ lines = f.read().split('#')
+ for line in lines:
+ line = line.strip()
+ keyword = base_path
+ index = line.find(keyword)
+ if (index < 0) or (len(line) == 0):
+ continue
+ else:
+ module_dir = line[index + len(keyword) + 1 : ]
+ if (len(module_dir) == 0):
+ continue
+ name_str = all_repository_set.get_repository_name(module_dir)
+ if (len(name_str) > 0):
+ prj_repository_set.add_repository((name_str, module_dir))
+ prj_repository_unique_set.add(name_str)
+ prj_repository_unique_set.add("rtos_sdk/build")
+ prj_repository_unique_set.add("rtos_sdk/scripts")
+
+ changed_reps_set = changed_reps.split('#')
+ changed_flag = 0
+ for changes_rep in changed_reps_set:
+ if changes_rep in prj_repository_unique_set:
+ changed_flag = 1
+ break
+
+ return changed_flag
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/repositories_changed_filter.py b/repositories_changed_filter.py
new file mode 100755
index 0000000..868adda
--- /dev/null
+++ b/repositories_changed_filter.py
@@ -0,0 +1,47 @@
+#!/usr/bin/python3
+#coding:utf-8
+#
+# Copyright (c) 2021-2022 Amlogic, Inc. All rights reserved.
+#
+# SPDX-License-Identifier: MIT
+#
+
+import subprocess
+
+def split_into_sections(content):
+ # Split the content by empty lines
+ paragraphs = content.split("\n\n")
+
+ # Initialize variables
+ sections = []
+ current_section = []
+
+ # Iterate over each paragraph
+ for paragraph in paragraphs:
+ lines = paragraph.split("\n")
+ if len(lines) >= 2 and lines[0].startswith("project") and lines[1].startswith("On branch"):
+ # Start a new section
+ if current_section:
+ sections.append(current_section)
+ current_section = [paragraph]
+ else:
+ # Add to the current section
+ current_section.append(paragraph)
+
+ # Add the last section
+ if current_section:
+ sections.append(current_section)
+
+ return sections
+
+shell_command = "repo forall -p -c git status"
+result = subprocess.run(shell_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+content = result.stdout
+
+sections = split_into_sections(content)
+changed_repository_list = ""
+for p in sections:
+ if p[0].find("is up to date with") < 0 and (p[0].find("git push") > 0 or p[0].find("different commits")) > 0:
+ reps_name = "rtos_sdk/" + p[0][0 + len("project") + 1 : p[0].find('\n') - 1]
+ changed_repository_list = changed_repository_list + reps_name + "#"
+print(changed_repository_list)
\ No newline at end of file