# Copyright (C) 2010 Google Inc. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following disclaimer # in the documentation and/or other materials provided with the # distribution. # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import json import logging import re import sys import traceback from testfile import TestFile JSON_RESULTS_FILE = "results.json" JSON_RESULTS_FILE_SMALL = "results-small.json" JSON_RESULTS_PREFIX = "ADD_RESULTS(" JSON_RESULTS_SUFFIX = ");" JSON_RESULTS_MIN_TIME = 3 JSON_RESULTS_HIERARCHICAL_VERSION = 4 JSON_RESULTS_MAX_BUILDS = 500 JSON_RESULTS_MAX_BUILDS_SMALL = 100 ACTUAL_KEY = "actual" BUG_KEY = "bugs" BUILD_NUMBERS_KEY = "buildNumbers" BUILDER_NAME_KEY = "builder_name" EXPECTED_KEY = "expected" FAILURE_MAP_KEY = "failure_map" FAILURES_BY_TYPE_KEY = "num_failures_by_type" FIXABLE_COUNTS_KEY = "fixableCounts" RESULTS_KEY = "results" TESTS_KEY = "tests" TIME_KEY = "time" TIMES_KEY = "times" VERSIONS_KEY = "version" AUDIO = "A" CRASH = "C" FAIL = "Q" # This is only output by gtests. FLAKY = "L" IMAGE = "I" IMAGE_PLUS_TEXT = "Z" MISSING = "O" NO_DATA = "N" NOTRUN = "Y" PASS = "P" SKIP = "X" TEXT = "F" TIMEOUT = "T" AUDIO_STRING = "AUDIO" CRASH_STRING = "CRASH" IMAGE_PLUS_TEXT_STRING = "IMAGE+TEXT" IMAGE_STRING = "IMAGE" FAIL_STRING = "FAIL" FLAKY_STRING = "FLAKY" MISSING_STRING = "MISSING" NO_DATA_STRING = "NO DATA" NOTRUN_STRING = "NOTRUN" PASS_STRING = "PASS" SKIP_STRING = "SKIP" TEXT_STRING = "TEXT" TIMEOUT_STRING = "TIMEOUT" FAILURE_TO_CHAR = { AUDIO_STRING: AUDIO, CRASH_STRING: CRASH, IMAGE_PLUS_TEXT_STRING: IMAGE_PLUS_TEXT, IMAGE_STRING: IMAGE, FLAKY_STRING: FLAKY, FAIL_STRING: FAIL, MISSING_STRING: MISSING, NO_DATA_STRING: NO_DATA, NOTRUN_STRING: NOTRUN, PASS_STRING: PASS, SKIP_STRING: SKIP, TEXT_STRING: TEXT, TIMEOUT_STRING: TIMEOUT, } # FIXME: Use dict comprehensions once we update the server to python 2.7. CHAR_TO_FAILURE = dict((value, key) for key, value in FAILURE_TO_CHAR.items()) def _is_directory(subtree): return RESULTS_KEY not in subtree class JsonResults(object): @classmethod def _strip_prefix_suffix(cls, data): if data.startswith(JSON_RESULTS_PREFIX) and data.endswith(JSON_RESULTS_SUFFIX): return data[len(JSON_RESULTS_PREFIX):len(data) - len(JSON_RESULTS_SUFFIX)] return data @classmethod def _generate_file_data(cls, jsonObject, sort_keys=False): return json.dumps(jsonObject, separators=(',', ':'), sort_keys=sort_keys) @classmethod def _load_json(cls, file_data): json_results_str = cls._strip_prefix_suffix(file_data) if not json_results_str: logging.warning("No json results data.") return None try: return json.loads(json_results_str) except: logging.debug(json_results_str) logging.error("Failed to load json results: %s", traceback.print_exception(*sys.exc_info())) return None @classmethod def _merge_json(cls, aggregated_json, incremental_json, num_runs): # We have to delete expected entries because the incremental json may not have any # entry for every test in the aggregated json. But, the incremental json will have # all the correct expected entries for that run. cls._delete_expected_entries(aggregated_json[TESTS_KEY]) cls._merge_non_test_data(aggregated_json, incremental_json, num_runs) incremental_tests = incremental_json[TESTS_KEY] if incremental_tests: aggregated_tests = aggregated_json[TESTS_KEY] cls._merge_tests(aggregated_tests, incremental_tests, num_runs) @classmethod def _delete_expected_entries(cls, aggregated_json): for key in aggregated_json: item = aggregated_json[key] if _is_directory(item): cls._delete_expected_entries(item) else: if EXPECTED_KEY in item: del item[EXPECTED_KEY] if BUG_KEY in item: del item[BUG_KEY] @classmethod def _merge_non_test_data(cls, aggregated_json, incremental_json, num_runs): incremental_builds = incremental_json[BUILD_NUMBERS_KEY] aggregated_builds = aggregated_json[BUILD_NUMBERS_KEY] aggregated_build_number = int(aggregated_builds[0]) # FIXME: It's no longer possible to have multiple runs worth of data in the incremental_json, # So we can get rid of this for-loop and the associated index. for index in reversed(range(len(incremental_builds))): build_number = int(incremental_builds[index]) logging.debug("Merging build %s, incremental json index: %d.", build_number, index) # Merge this build into aggreagated results. cls._merge_one_build(aggregated_json, incremental_json, index, num_runs) @classmethod def _merge_one_build(cls, aggregated_json, incremental_json, incremental_index, num_runs): for key in incremental_json.keys(): # Merge json results except "tests" properties (results, times etc). # "tests" properties will be handled separately. if key == TESTS_KEY or key == FAILURE_MAP_KEY: continue if key in aggregated_json: if key == FAILURES_BY_TYPE_KEY: cls._merge_one_build(aggregated_json[key], incremental_json[key], incremental_index, num_runs=num_runs) else: aggregated_json[key].insert(0, incremental_json[key][incremental_index]) aggregated_json[key] = aggregated_json[key][:num_runs] else: aggregated_json[key] = incremental_json[key] @classmethod def _merge_tests(cls, aggregated_json, incremental_json, num_runs): # FIXME: Some data got corrupted and has results/times at the directory level. # Once the data is fixe, this should assert that the directory level does not have # results or times and just return "RESULTS_KEY not in subtree". if RESULTS_KEY in aggregated_json: del aggregated_json[RESULTS_KEY] if TIMES_KEY in aggregated_json: del aggregated_json[TIMES_KEY] all_tests = set(aggregated_json.iterkeys()) if incremental_json: all_tests |= set(incremental_json.iterkeys()) for test_name in all_tests: if test_name not in aggregated_json: aggregated_json[test_name] = incremental_json[test_name] continue incremental_sub_result = incremental_json[test_name] if incremental_json and test_name in incremental_json else None if _is_directory(aggregated_json[test_name]): cls._merge_tests(aggregated_json[test_name], incremental_sub_result, num_runs) continue aggregated_test = aggregated_json[test_name] if incremental_sub_result: results = incremental_sub_result[RESULTS_KEY] times = incremental_sub_result[TIMES_KEY] if EXPECTED_KEY in incremental_sub_result and incremental_sub_result[EXPECTED_KEY] != PASS_STRING: aggregated_test[EXPECTED_KEY] = incremental_sub_result[EXPECTED_KEY] if BUG_KEY in incremental_sub_result: aggregated_test[BUG_KEY] = incremental_sub_result[BUG_KEY] else: results = [[1, NO_DATA]] times = [[1, 0]] cls._insert_item_run_length_encoded(results, aggregated_test[RESULTS_KEY], num_runs) cls._insert_item_run_length_encoded(times, aggregated_test[TIMES_KEY], num_runs) @classmethod def _insert_item_run_length_encoded(cls, incremental_item, aggregated_item, num_runs): for item in incremental_item: if len(aggregated_item) and item[1] == aggregated_item[0][1]: aggregated_item[0][0] = min(aggregated_item[0][0] + item[0], num_runs) else: aggregated_item.insert(0, item) @classmethod def _normalize_results(cls, aggregated_json, num_runs, run_time_pruning_threshold): names_to_delete = [] for test_name in aggregated_json: if _is_directory(aggregated_json[test_name]): cls._normalize_results(aggregated_json[test_name], num_runs, run_time_pruning_threshold) # If normalizing deletes all the children of this directory, also delete the directory. if not aggregated_json[test_name]: names_to_delete.append(test_name) else: leaf = aggregated_json[test_name] leaf[RESULTS_KEY] = cls._remove_items_over_max_number_of_builds(leaf[RESULTS_KEY], num_runs) leaf[TIMES_KEY] = cls._remove_items_over_max_number_of_builds(leaf[TIMES_KEY], num_runs) if cls._should_delete_leaf(leaf, run_time_pruning_threshold): names_to_delete.append(test_name) for test_name in names_to_delete: del aggregated_json[test_name] @classmethod def _should_delete_leaf(cls, leaf, run_time_pruning_threshold): if leaf.get(EXPECTED_KEY, PASS_STRING) != PASS_STRING: return False if BUG_KEY in leaf: return False deletable_types = set((PASS, NO_DATA, NOTRUN)) for result in leaf[RESULTS_KEY]: if result[1] not in deletable_types: return False for time in leaf[TIMES_KEY]: if time[1] >= run_time_pruning_threshold: return False return True @classmethod def _remove_items_over_max_number_of_builds(cls, encoded_list, num_runs): num_builds = 0 index = 0 for result in encoded_list: num_builds = num_builds + result[0] index = index + 1 if num_builds >= num_runs: return encoded_list[:index] return encoded_list @classmethod def _convert_gtest_json_to_aggregate_results_format(cls, json): # FIXME: Change gtests over to uploading the full results format like layout-tests # so we don't have to do this normalizing. # http://crbug.com/247192. if FAILURES_BY_TYPE_KEY in json: # This is already in the right format. return failures_by_type = {} for fixableCount in json[FIXABLE_COUNTS_KEY]: for failure_type, count in fixableCount.items(): failure_string = CHAR_TO_FAILURE[failure_type] if failure_string not in failures_by_type: failures_by_type[failure_string] = [] failures_by_type[failure_string].append(count) json[FAILURES_BY_TYPE_KEY] = failures_by_type @classmethod def _check_json(cls, builder, json): version = json[VERSIONS_KEY] if version > JSON_RESULTS_HIERARCHICAL_VERSION: return "Results JSON version '%s' is not supported." % version if not builder in json: return "Builder '%s' is not in json results." % builder results_for_builder = json[builder] if not BUILD_NUMBERS_KEY in results_for_builder: return "Missing build number in json results." cls._convert_gtest_json_to_aggregate_results_format(json[builder]) # FIXME: Remove this once all the bots have cycled with this code. # The failure map was moved from the top-level to being below the builder # like everything else. if FAILURE_MAP_KEY in json: del json[FAILURE_MAP_KEY] # FIXME: Remove this code once the gtests switch over to uploading the full_results.json format. # Once the bots have cycled with this code, we can move this loop into _convert_gtest_json_to_aggregate_results_format. KEYS_TO_DELETE = ["fixableCount", "fixableCounts", "allFixableCount"] for key in KEYS_TO_DELETE: if key in json[builder]: del json[builder][key] return "" @classmethod def _populate_tests_from_full_results(cls, full_results, new_results): if EXPECTED_KEY in full_results: expected = full_results[EXPECTED_KEY] if expected != PASS_STRING and expected != NOTRUN_STRING: new_results[EXPECTED_KEY] = expected time = int(round(full_results[TIME_KEY])) if TIME_KEY in full_results else 0 new_results[TIMES_KEY] = [[1, time]] actual_failures = full_results[ACTUAL_KEY] # Treat unexpected skips like NOTRUNs to avoid exploding the results JSON files # when a bot exits early (e.g. due to too many crashes/timeouts). if expected != SKIP_STRING and actual_failures == SKIP_STRING: expected = first_actual_failure = NOTRUN_STRING elif expected == NOTRUN_STRING: first_actual_failure = expected else: # FIXME: Include the retry result as well and find a nice way to display it in the flakiness dashboard. first_actual_failure = actual_failures.split(' ')[0] new_results[RESULTS_KEY] = [[1, FAILURE_TO_CHAR[first_actual_failure]]] if BUG_KEY in full_results: new_results[BUG_KEY] = full_results[BUG_KEY] return for key in full_results: new_results[key] = {} cls._populate_tests_from_full_results(full_results[key], new_results[key]) @classmethod def _convert_full_results_format_to_aggregate(cls, full_results_format): num_total_tests = 0 num_failing_tests = 0 failures_by_type = full_results_format[FAILURES_BY_TYPE_KEY] tests = {} cls._populate_tests_from_full_results(full_results_format[TESTS_KEY], tests) aggregate_results_format = { VERSIONS_KEY: JSON_RESULTS_HIERARCHICAL_VERSION, full_results_format[BUILDER_NAME_KEY]: { # FIXME: Use dict comprehensions once we update the server to python 2.7. FAILURES_BY_TYPE_KEY: dict((key, [value]) for key, value in failures_by_type.items()), TESTS_KEY: tests, # FIXME: Have all the consumers of this switch over to the full_results_format keys # so we don't have to do this silly conversion. Or switch the full_results_format keys # to be camel-case. BUILD_NUMBERS_KEY: [full_results_format['build_number']], 'chromeRevision': [full_results_format['chromium_revision']], 'blinkRevision': [full_results_format['blink_revision']], 'secondsSinceEpoch': [full_results_format['seconds_since_epoch']], } } return aggregate_results_format @classmethod def _get_incremental_json(cls, builder, incremental_string, is_full_results_format): if not incremental_string: return "No incremental JSON data to merge.", 403 logging.info("Loading incremental json.") incremental_json = cls._load_json(incremental_string) if not incremental_json: return "Incremental JSON data is not valid JSON.", 403 if is_full_results_format: logging.info("Converting full results format to aggregate.") incremental_json = cls._convert_full_results_format_to_aggregate(incremental_json) logging.info("Checking incremental json.") check_json_error_string = cls._check_json(builder, incremental_json) if check_json_error_string: return check_json_error_string, 403 return incremental_json, 200 @classmethod def _get_aggregated_json(cls, builder, aggregated_string): logging.info("Loading existing aggregated json.") aggregated_json = cls._load_json(aggregated_string) if not aggregated_json: return None, 200 logging.info("Checking existing aggregated json.") check_json_error_string = cls._check_json(builder, aggregated_json) if check_json_error_string: return check_json_error_string, 500 return aggregated_json, 200 @classmethod def merge(cls, builder, aggregated_string, incremental_json, num_runs, sort_keys=False): aggregated_json, status_code = cls._get_aggregated_json(builder, aggregated_string) if not aggregated_json: aggregated_json = incremental_json elif status_code != 200: return aggregated_json, status_code else: if aggregated_json[builder][BUILD_NUMBERS_KEY][0] == incremental_json[builder][BUILD_NUMBERS_KEY][0]: status_string = "Incremental JSON's build number %s is the latest build number in the aggregated JSON." % str(aggregated_json[builder][BUILD_NUMBERS_KEY][0]) return status_string, 409 logging.info("Merging json results.") try: cls._merge_json(aggregated_json[builder], incremental_json[builder], num_runs) except: return "Failed to merge json results: %s", traceback.print_exception(*sys.exc_info()), 500 aggregated_json[VERSIONS_KEY] = JSON_RESULTS_HIERARCHICAL_VERSION aggregated_json[builder][FAILURE_MAP_KEY] = CHAR_TO_FAILURE is_debug_builder = re.search(r"(Debug|Dbg)", builder, re.I) run_time_pruning_threshold = 3 * JSON_RESULTS_MIN_TIME if is_debug_builder else JSON_RESULTS_MIN_TIME cls._normalize_results(aggregated_json[builder][TESTS_KEY], num_runs, run_time_pruning_threshold) return cls._generate_file_data(aggregated_json, sort_keys), 200 @classmethod def _get_file(cls, master, builder, test_type, filename): files = TestFile.get_files(master, builder, test_type, filename) if files: return files[0] file = TestFile() file.master = master file.builder = builder file.test_type = test_type file.name = filename file.data = "" return file @classmethod def update(cls, master, builder, test_type, incremental_string, is_full_results_format): logging.info("Updating %s and %s." % (JSON_RESULTS_FILE_SMALL, JSON_RESULTS_FILE)) small_file = cls._get_file(master, builder, test_type, JSON_RESULTS_FILE_SMALL) large_file = cls._get_file(master, builder, test_type, JSON_RESULTS_FILE) return cls.update_files(builder, incremental_string, small_file, large_file, is_full_results_format) @classmethod def update_files(cls, builder, incremental_string, small_file, large_file, is_full_results_format): incremental_json, status_code = cls._get_incremental_json(builder, incremental_string, is_full_results_format) if status_code != 200: return incremental_json, status_code status_string, status_code = cls.update_file(builder, small_file, incremental_json, JSON_RESULTS_MAX_BUILDS_SMALL) if status_code != 200: return status_string, status_code return cls.update_file(builder, large_file, incremental_json, JSON_RESULTS_MAX_BUILDS) @classmethod def update_file(cls, builder, file, incremental_json, num_runs): new_results, status_code = cls.merge(builder, file.data, incremental_json, num_runs) if status_code != 200: return new_results, status_code return TestFile.save_file(file, new_results) @classmethod def _delete_results_and_times(cls, tests): for key in tests.keys(): if key in (RESULTS_KEY, TIMES_KEY): del tests[key] else: cls._delete_results_and_times(tests[key]) @classmethod def get_test_list(cls, builder, json_file_data): logging.debug("Loading test results json...") json = cls._load_json(json_file_data) if not json: return None logging.debug("Checking test results json...") check_json_error_string = cls._check_json(builder, json) if check_json_error_string: return None test_list_json = {} tests = json[builder][TESTS_KEY] cls._delete_results_and_times(tests) test_list_json[builder] = {TESTS_KEY: tests} return cls._generate_file_data(test_list_json)