diff --git a/.github/protected-workflows.yml b/.github/protected-workflows.yml index da5460b3d23..bc2ced16a73 100644 --- a/.github/protected-workflows.yml +++ b/.github/protected-workflows.yml @@ -15,4 +15,4 @@ events: anyEvent: trustAnyone: false trustCollaborators: false - trustedUserNames: [] \ No newline at end of file + trustedUserNames: [] diff --git a/.github/scripts/check-lineends.py b/.github/scripts/check-lineends.py new file mode 100755 index 00000000000..5537f348384 --- /dev/null +++ b/.github/scripts/check-lineends.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +import argparse +import re +import sys + +ANY_NEWLINE = re.compile(rb'(\r\n|\r|\n)') + +parser = argparse.ArgumentParser( + prog='check-lineends.py', + description='Line end checker for the MMTk project', + epilog=''' +This script checks if the file FILENAME has proper line ends: + +1. it uses UNIX line ends, and +2. it has a newline character at the end of the file + +If you add the -f option, it will try to fix the line ends if wrong. +''') + +parser.add_argument('-q', '--quiet', action='store_true', help='Quiet mode') +parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') +parser.add_argument('-f', '--fix', action='store_true', help='Fix files with wrong line ends') +parser.add_argument('filename', nargs='*', help='File name') + +verbosity = 1 + +def pv(level, *args, **kwargs): + if verbosity >= level: + print(*args, **kwargs) + +def process_file(filename, fix): + pv(2, "Processing file:", filename) + with open(filename, 'rb') as f: + content = f.read() + + non_unix = b'\r' in content + no_eol = not content.endswith(b'\n') + wrong = non_unix or no_eol + + if non_unix: + pv(1, "File contains non-UNIX line ends:", filename) + if no_eol: + pv(1, "File does not end with a newline character:", filename) + + if wrong and fix: + pv(1, "Fixing file:", filename) + fixed_content = ANY_NEWLINE.sub(b'\n', content) + if no_eol: + fixed_content += b'\n' + with open(filename, 'wb') as f: + f.write(fixed_content) + + return wrong + + +def main(): + args = parser.parse_args() + + global verbosity + if args.quiet: + verbosity = 0 + if args.verbose: + verbosity = 2 + + any_wrong = False + + for filename in args.filename: + if process_file(filename, args.fix) == True: + any_wrong = True + + if any_wrong: + sys.exit(1) + +if __name__=='__main__': + main() diff --git a/.github/scripts/ci-check-lineends.sh b/.github/scripts/ci-check-lineends.sh new file mode 100755 index 00000000000..7eed732621e --- /dev/null +++ b/.github/scripts/ci-check-lineends.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +# This is a driver script for check-lineends.py +# It finds text files in the project tree and checks/fixes its line ends. +# +# The CI runs this script during style checking. +# +# Developers may also run this script directly. +# It forwards all command line options to check-lineends.py. +# This means if you add the '-f' option, +# it will automatically fix the line ends of all files we concern. +# +# ./.github/scripts/ci-check-lineends.sh -f +# +# You can also pass the '-v' option to see which files it is checking. +# +# ./.github/scripts/ci-check-lineends.sh -v +# +# In this project, text files use UNIX line ends, and must have a newline character at the end of the file. +# Note that not having a newline character at the end of a file may have unexpected consequences. +# For example, when concatenating multiple files, +# the last line of a file will be joined with the first line of the next file. +# The same may happen when including files using `#include` or `include!` directives in C or Rust. + +BAD_LINE_ENDS=0 + +# TODO: When we introduce the '.gitattributes' file, +# make sure the patterns here matches the patterns in '.gitattributes'. +# Alternatively, find a way to automatically establish the list of files to check +# from the contents of '.gitattributes'. +FILES=$(find . -name 'target' -prune -o -type f -a '(' \ + -name '.gitignore' \ + -o -name '*.rs' \ + -o -name '*.h' \ + -o -name '*.yml' \ + -o -name '*.sh' \ + -o -name '*.toml' \ + -o -name '*.lock' \ + -o -name '*.py' \ + -o -name '*.bt' \ + -o -name '*.bt.fragment' \ + -o -name '*.md' \ + -o -name '*.html' \ + -o -name '*.css' \ + -o -name '*.js' \ + -o -name 'COPYRIGHT' \ + -o -name 'LICENSE-*' \ + -o -name 'rust-toolchain' \ + -o -name '.gitignore' \ + ')' -print) + +if ! xargs $(dirname $0)/check-lineends.py "$@" <<<$FILES; then + BAD_LINE_ENDS=1 +fi + + +if [[ "$BAD_LINE_ENDS" -ne 0 ]]; then + echo "ERROR: Some text files have non-unix line ends or do not have newline character at the end of file." + exit 1 +fi diff --git a/.github/scripts/ci-style.sh b/.github/scripts/ci-style.sh index 0482321b521..05138dbb6b4 100755 --- a/.github/scripts/ci-style.sh +++ b/.github/scripts/ci-style.sh @@ -2,6 +2,13 @@ export RUSTFLAGS="-D warnings -A unknown-lints" +# --- Check line ends of text files --- + +if ! $project_root/.github/scripts/ci-check-lineends.sh; then + echo "ERROR: Line ends check failed." + exit 1 +fi + # --- Check format --- cargo fmt -- --check cargo fmt --manifest-path=macros/Cargo.toml -- --check diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 45685a5da32..dbaa67c4823 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,4 +15,4 @@ Generally we expect a pull request to meeting the following requirements before 2. The code is well documented. 3. The PR does not introduce unsafe Rust code unless necessary. Whenever introducing unsafe code, the contributor must elaborate why it is necessary. 4. The PR passes the mmtk-core unit tests and complies with the coding style. We have scripts in `.github/scripts` that are used by our Github action to run those checks for each PR. -5. The PR passes all the binding tests. We run benchmarks with bindings to test mmtk-core. A new pull request should not break bindings, as we ensure that our supported bindings always work with the latest mmtk-core. If a pull request makes changes that require the bindings to be updated correspondingly, you can approach the MMTk team on [our Zulip](https://mmtk.zulipchat.com/) and seek help from them to update the bindings. \ No newline at end of file +5. The PR passes all the binding tests. We run benchmarks with bindings to test mmtk-core. A new pull request should not break bindings, as we ensure that our supported bindings always work with the latest mmtk-core. If a pull request makes changes that require the bindings to be updated correspondingly, you can approach the MMTk team on [our Zulip](https://mmtk.zulipchat.com/) and seek help from them to update the bindings. diff --git a/LICENSE-MIT b/LICENSE-MIT index 468cd79a8f6..31aa79387f2 100644 --- a/LICENSE-MIT +++ b/LICENSE-MIT @@ -20,4 +20,4 @@ SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. \ No newline at end of file +DEALINGS IN THE SOFTWARE. diff --git a/docs/userguide/src/README.md b/docs/userguide/src/README.md index 032a56b2222..0ee654d70da 100644 --- a/docs/userguide/src/README.md +++ b/docs/userguide/src/README.md @@ -4,4 +4,4 @@ MMTk is a memory management toolkit providing language implementers with a power and researchers with a multi-runtime platform for memory management research. It is a complete re-write of the original MMTk, which was written in Java as part of Jikes RVM. - \ No newline at end of file + diff --git a/docs/userguide/src/portingguide/before_start.md b/docs/userguide/src/portingguide/before_start.md index f70c471baf4..cad4342bbee 100644 --- a/docs/userguide/src/portingguide/before_start.md +++ b/docs/userguide/src/portingguide/before_start.md @@ -12,4 +12,4 @@ Key questions include: - Does the runtime support precise stack scanning? - etc. -Thinking through these questions should give you a sense for how big a task a GC port will be. \ No newline at end of file +Thinking through these questions should give you a sense for how big a task a GC port will be. diff --git a/docs/userguide/src/portingguide/portability.md b/docs/userguide/src/portingguide/portability.md index e1ffd7bac71..3825d0198f2 100644 --- a/docs/userguide/src/portingguide/portability.md +++ b/docs/userguide/src/portingguide/portability.md @@ -28,4 +28,4 @@ The leftmost box should be entirely free of any MMTk-specific code. > Note: we do currently maintain a fork of OpenJDK which includes some necessary changes to their code base, but this is not MMTk-specific and ideally this will be upstreamed. Our port to V8 is a cleaner example, where we’ve managed to work closely with the V8 team to upstream all of the refactoring of the V8 code base that was necessary for it to support a third party heap. -We structure the code into three repos. Taking the example of the OpenJDK port, the three repos are: the [MMTk core](https://github.com/mmtk/mmtk-core), the [binding repo](https://github.com/mmtk/mmtk-openjdk) containing both parts of the binding, and the OpenJDK repo, which is currently [a fork](https://github.com/mmtk/openjdk) we maintain. \ No newline at end of file +We structure the code into three repos. Taking the example of the OpenJDK port, the three repos are: the [MMTk core](https://github.com/mmtk/mmtk-core), the [binding repo](https://github.com/mmtk/mmtk-openjdk) containing both parts of the binding, and the OpenJDK repo, which is currently [a fork](https://github.com/mmtk/openjdk) we maintain. diff --git a/docs/userguide/src/portingguide/prefix.md b/docs/userguide/src/portingguide/prefix.md index c583f9f6ea8..dd93f2a62f9 100644 --- a/docs/userguide/src/portingguide/prefix.md +++ b/docs/userguide/src/portingguide/prefix.md @@ -5,4 +5,4 @@ This guide is designed to get you started on porting MMTk to a new runtime. We start with an overview of the MMTk approach to porting and then step through recommended strategies for implementing a port. There’s no fixed way to implement a new port. -What we outline here is a distillation of best practices that have emerged from community as it has worked through many ports (each at various levels of maturity). \ No newline at end of file +What we outline here is a distillation of best practices that have emerged from community as it has worked through many ports (each at various levels of maturity). diff --git a/docs/userguide/src/tutorial/further_reading.md b/docs/userguide/src/tutorial/further_reading.md index 3f0f3ae9185..0c092d8b247 100644 --- a/docs/userguide/src/tutorial/further_reading.md +++ b/docs/userguide/src/tutorial/further_reading.md @@ -5,4 +5,4 @@ - [*Oil and Water? High Performance Garbage Collection in Java with MMTk*](https://www.mmtk.io/assets/pubs/mmtk-icse-2004.pdf) (Blackburn, Cheng, McKinley, 2004) - [*Myths and realities: The performance impact of garbage collection*](https://www.mmtk.io/assets/pubs/mmtk-sigmetrics-2004.pdf) (Blackburn, Cheng, McKinley, 2004) - [*The Garbage Collection Handbook*](https://learning.oreilly.com/library/view/the-garbage-collection/9781315388007) (Jones, Hosking, Moss, 2016) -- Videos: [MPLR 2020 Keynote](https://www.youtube.com/watch?v=3L6XEVaYAmU), [Deconstructing the Garbage-First Collector](https://www.youtube.com/watch?v=MAk6RdApGLs) \ No newline at end of file +- Videos: [MPLR 2020 Keynote](https://www.youtube.com/watch?v=3L6XEVaYAmU), [Deconstructing the Garbage-First Collector](https://www.youtube.com/watch?v=MAk6RdApGLs) diff --git a/docs/userguide/src/tutorial/intro/glossary.md b/docs/userguide/src/tutorial/intro/glossary.md index 2da6cd37a85..f7c670f1968 100644 --- a/docs/userguide/src/tutorial/intro/glossary.md +++ b/docs/userguide/src/tutorial/intro/glossary.md @@ -52,4 +52,4 @@ what this tutorial covers. On the other hand, someone wishing to introduce an entirely new garbage collection policy (such as Immix, for example), would need to first create a policy which specifies that algorithm, before creating a plan which defines -how the GC algorithm fits together and utilizes that policy. \ No newline at end of file +how the GC algorithm fits together and utilizes that policy. diff --git a/docs/userguide/src/tutorial/intro/what_is_mmtk.md b/docs/userguide/src/tutorial/intro/what_is_mmtk.md index a0049f82ab2..7e74b93c2f7 100644 --- a/docs/userguide/src/tutorial/intro/what_is_mmtk.md +++ b/docs/userguide/src/tutorial/intro/what_is_mmtk.md @@ -11,4 +11,4 @@ very flexible with runtime and able to be ported to many different VMs. The principal idea of MMTk is that it can be used as a toolkit, allowing new GC algorithms to be rapidly developed using common components. It also allows different GC algorithms to be -compared on an apples-to-apples basis, since they share common mechanisms. \ No newline at end of file +compared on an apples-to-apples basis, since they share common mechanisms. diff --git a/docs/userguide/src/tutorial/mygc/ss/exercise.md b/docs/userguide/src/tutorial/mygc/ss/exercise.md index e91100c36b8..37c72ccfa70 100644 --- a/docs/userguide/src/tutorial/mygc/ss/exercise.md +++ b/docs/userguide/src/tutorial/mygc/ss/exercise.md @@ -25,4 +25,4 @@ next thing to do is to make this collector into a more efficient proper generational collector. When you are finished, try running the benchmarks and seeing how the -performance of this collector compares to MyGC. Great work! \ No newline at end of file +performance of this collector compares to MyGC. Great work! diff --git a/docs/userguide/src/tutorial/prefix.md b/docs/userguide/src/tutorial/prefix.md index 2f4dd8f7411..bb718ff5946 100644 --- a/docs/userguide/src/tutorial/prefix.md +++ b/docs/userguide/src/tutorial/prefix.md @@ -15,4 +15,4 @@ This tutorial is a work in progress. Some sections may be rough, and others may be missing information (especially about import statements). If something is missing or inaccurate, refer to the relevant completed garbage collector if possible. Please also raise an issue, or create a pull request addressing -the problem. \ No newline at end of file +the problem. diff --git a/docs/userguide/src/tutorial/preliminaries/test.md b/docs/userguide/src/tutorial/preliminaries/test.md index 8ce254eaf8b..e9fc71036ac 100644 --- a/docs/userguide/src/tutorial/preliminaries/test.md +++ b/docs/userguide/src/tutorial/preliminaries/test.md @@ -121,4 +121,4 @@ pass, as garbage will be collected, and the smaller benchmarks should run the same as they did while using NoGC. ``` MMTK_PLAN=SemiSpace ./build/linux-x86_64-normal-server-$DEBUG_LEVEL/jdk/bin/java -XX:+UseThirdPartyHeap -Xms512M -Xmx512M -jar ./dacapo-9.12-MR1-bach.jar lusearch - ``` \ No newline at end of file + ```