From cfa650a4a712f4c6fe16639ad36ce22824b63801 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Fri, 21 May 2021 12:04:33 -0700 Subject: [PATCH 1/4] Switch CI to GitHub Actions --- .github/workflows/ci.yml | 49 ++++++++++++++++++++++++++++++++++++++++ .travis.yml | 20 ---------------- README.md | 4 +--- appveyor.yml | 42 ---------------------------------- 4 files changed, 50 insertions(+), 65 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .travis.yml delete mode 100644 appveyor.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0ac32ad --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,49 @@ +name: CI +on: + push: + branches: [master] + tags: ["*"] + pull_request: +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1.0' + - '1' # automatically expands to the latest stable 1.x release of Julia + - 'nightly' + os: + - ubuntu-latest + - macOS-latest + - windows-latest + arch: + - x64 + - x86 + exclude: + - os: macOS-latest + arch: x86 + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: actions/cache@v1 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v1 + with: + file: lcov.info diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 3aade56..0000000 --- a/.travis.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Documentation: http://docs.travis-ci.com/user/languages/julia/ -language: julia -os: - - linux - - osx -julia: - - 1.0 - - 1 - - nightly -notifications: - email: false -# uncomment the following lines to override the default test script -#script: -# - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi -# - julia -e 'Pkg.clone(pwd()); Pkg.build("StringEncodings"); Pkg.test("StringEncodings"; coverage=true)' -after_success: - # push coverage results to Coveralls - - julia -e 'cd(Pkg.dir("StringEncodings")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())' - # push coverage results to Codecov - - julia -e 'cd(Pkg.dir("StringEncodings")); Pkg.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())' diff --git a/README.md b/README.md index d4dd847..b5e2552 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ # StringEncodings -[![Travis CI Build Status](https://travis-ci.org/JuliaStrings/StringEncodings.jl.svg?branch=master)](https://travis-ci.org/JuliaStrings/StringEncodings.jl) -[![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/3gslhfg91isldnvq?svg=true)](https://ci.appveyor.com/project/nalimilan/stringencodings-jl) -[![Coveralls Coverage Status](https://coveralls.io/repos/JuliaStrings/StringEncodings.jl/badge.svg?branch=master&service=github)](https://coveralls.io/github/JuliaStrings/StringEncodings.jl?branch=master) +[![Build status](https://github.com/JuliaStrings/StringEncodings.jl/workflows/CI/badge.svg)](https://github.com/JuliaStrings/StringEncodings.jl/actions?query=workflow%3ACI+branch%3Amaster) [![Codecov Coverage Status](http://codecov.io/github/JuliaStrings/StringEncodings.jl/coverage.svg?branch=master)](http://codecov.io/github/JuliaStrings/StringEncodings.jl?branch=master) This Julia package provides support for decoding and encoding texts between multiple character encodings. It is currently based on the iconv interface, and supports all major platforms using either the native iconv support or [GNU libiconv](https://www.gnu.org/software/libiconv/). In the future, native Julia support for major encodings will be added. diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index e22f573..0000000 --- a/appveyor.yml +++ /dev/null @@ -1,42 +0,0 @@ -environment: - matrix: - - julia_version: 1.0 - - julia_version: 1 - - julia_version: nightly - -platform: - - x86 # 32-bit - - x64 # 64-bit - -matrix: - allow_failures: - - julia_version: nightly - -branches: - only: - - master - - /release-.*/ - - /v(\d+)\.(\d+)\.(\d+)/ - -notifications: - - provider: Email - on_build_success: false - on_build_failure: false - on_build_status_changed: false - -install: - - ps: iex ((new-object net.webclient).DownloadString("https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/version-1/bin/install.ps1")) - -build_script: - - echo "%JL_BUILD_SCRIPT%" - - C:\julia\bin\julia -e "%JL_BUILD_SCRIPT%" - -test_script: - - echo "%JL_TEST_SCRIPT%" - - C:\julia\bin\julia -e "%JL_TEST_SCRIPT%" - -# # Uncomment to support code coverage upload. Should only be enabled for packages -# # which would have coverage gaps without running on Windows -# on_success: -# - echo "%JL_CODECOV_SCRIPT%" -# - C:\julia\bin\julia -e "%JL_CODECOV_SCRIPT%" From e6bd7776226bfb957ffd017a751c201027fa0193 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Fri, 21 May 2021 12:01:42 -0700 Subject: [PATCH 2/4] Use BinaryBuilder-provided iconv Fixes #36. --- .github/workflows/ci.yml | 2 +- .gitignore | 8 +-- Project.toml | 7 +-- README.md | 6 +- deps/build.jl | 124 --------------------------------------- src/StringEncodings.jl | 28 ++------- 6 files changed, 12 insertions(+), 163 deletions(-) delete mode 100644 deps/build.jl diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ac32ad..05acec2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: fail-fast: false matrix: version: - - '1.0' + - '1.3' - '1' # automatically expands to the latest stable 1.x release of Julia - 'nightly' os: diff --git a/.gitignore b/.gitignore index c733316..3f02ca7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,4 @@ *.jl.cov *.jl.*.cov *.jl.mem -deps/builds -deps/downloads -deps/src -deps/usr -deps/deps.jl -deps/build.log -MANIFEST.TOML +Manifest.toml diff --git a/Project.toml b/Project.toml index 2f03385..e8cc82b 100644 --- a/Project.toml +++ b/Project.toml @@ -3,12 +3,11 @@ uuid = "69024149-9ee7-55f6-a4c4-859efe599b68" version = "0.3.3" [deps] -BinaryProvider = "b99e7846-7c00-51b0-8f62-c81ae34c0232" -Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +Libiconv_jll = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" [compat] -BinaryProvider = "0.4.1, 0.5" -julia = "0.7, 1" +Libiconv_jll = "1.16" +julia = "1.3" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/README.md b/README.md index b5e2552..9438571 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Build status](https://github.com/JuliaStrings/StringEncodings.jl/workflows/CI/badge.svg)](https://github.com/JuliaStrings/StringEncodings.jl/actions?query=workflow%3ACI+branch%3Amaster) [![Codecov Coverage Status](http://codecov.io/github/JuliaStrings/StringEncodings.jl/coverage.svg?branch=master)](http://codecov.io/github/JuliaStrings/StringEncodings.jl?branch=master) -This Julia package provides support for decoding and encoding texts between multiple character encodings. It is currently based on the iconv interface, and supports all major platforms using either the native iconv support or [GNU libiconv](https://www.gnu.org/software/libiconv/). In the future, native Julia support for major encodings will be added. +This Julia package provides support for decoding and encoding texts between multiple character encodings. It is currently based on the iconv interface, and supports all major platforms using [GNU libiconv](https://www.gnu.org/software/libiconv/). In the future, native Julia support for major encodings will be added. ## Encoding and Decoding Strings *Encoding* a refers to the process of converting a string (of any `AbstractString` type) to a sequence of bytes represented as a `Vector{UInt8}`. *Decoding* refers to the inverse process. @@ -151,7 +151,3 @@ julia> read(s, String) # Decoding happens automatically here Do not forget to call `close` on `StringEncoder` and `StringDecoder` objects to finish the encoding process. For `StringEncoder`, this function calls `flush`, which writes any characters still in the buffer, and possibly some control sequences (for stateful encodings). For both `StringEncoder` and `StringDecoder`, `close` checks that there are no incomplete sequences left in the input stream, and raise an `IncompleteSequenceError` if that's the case. It will also free iconv resources immediately, instead of waiting for garbage collection. Conversion currently raises an error if an invalid byte sequence is encountered in the input, or if some characters cannot be represented in the target enconding. It is not yet possible to ignore such characters or to replace them with a placeholder. - -## Notes on Installation on Linux OS - -Most Linux distributions provide `iconv` functionalities as part of the base operating system library `glibc`. In normal circumstances, no additional installation of `libiconv` should be required. If you observe such a behavior on your operating system, file an issue with OS details. diff --git a/deps/build.jl b/deps/build.jl deleted file mode 100644 index ea176ec..0000000 --- a/deps/build.jl +++ /dev/null @@ -1,124 +0,0 @@ -using Libdl -# Check for an iconv implementation with the GNU (non-POSIX) behavior: -# EILSEQ is returned when a sequence cannot be converted to target encoding, -# instead of succeeding and only returning the number of invalid conversions -# This non-standard behavior is required to allow replacing invalid sequences -# with a user-defined character. -# Implementations with this behavior include glibc, GNU libiconv (on which Mac -# OS X's is based) and win_iconv. -function validate_iconv(lib, iconv_open, iconv_close, iconv) - h = dlopen_e(lib) - h == C_NULL && return false - # Needed to check libc - f = dlsym_e(h, iconv_open) - f == C_NULL && return false - - cd = ccall(f, Ptr{Nothing}, (Cstring, Cstring), "ASCII", "UTF-8") - cd == Ptr{Nothing}(-1) && return false - - s = "café" - a = Vector{UInt8}(undef, sizeof(s)) - inbufptr = Ref{Ptr{UInt8}}(pointer(s)) - inbytesleft = Ref{Csize_t}(sizeof(s)) - outbufptr = Ref{Ptr{UInt8}}(pointer(a)) - outbytesleft = Ref{Csize_t}(length(a)) - ret = ccall(dlsym_e(h, iconv), Csize_t, - (Ptr{Nothing}, Ptr{Ptr{UInt8}}, Ref{Csize_t}, Ptr{Ptr{UInt8}}, Ref{Csize_t}), - cd, inbufptr, inbytesleft, outbufptr, outbytesleft) - ccall(dlsym_e(h, iconv_close), Nothing, (Ptr{Nothing},), cd) == -1 && return false - - return ret == -1 % Csize_t && Libc.errno() == Libc.EILSEQ -end - -using BinaryProvider - -# Parse some basic command-line arguments -const verbose = "--verbose" in ARGS -const prefix = Prefix(get([a for a in ARGS if a != "--verbose"], 1, joinpath(@__DIR__, "usr"))) -products = [ - LibraryProduct(prefix, String["libiconv"], :libiconv) -] - -# Download binaries from hosted location -bin_prefix = "https://github.com/JuliaStrings/IConvBuilder/releases/download/v1.15+build.3" - -# Listing of files generated by BinaryBuilder: -download_info = Dict( - Linux(:aarch64, :glibc) => ("$bin_prefix/IConv.aarch64-linux-gnu.tar.gz", "7a8d86f866ebee3218d590f74887d464c9980125a2e0c590acdcc0cfc5c84dd5"), - Linux(:aarch64, :musl) => ("$bin_prefix/IConv.aarch64-linux-musl.tar.gz", "a37fb558319c37970e70bdc786d6bfe84f95ca27a72f6d334a1411d9c305ee2c"), - Linux(:armv7l, :glibc, :eabihf) => ("$bin_prefix/IConv.arm-linux-gnueabihf.tar.gz", "c3e7eaf54fe3722153e29e29f1e08f7acdbe432fb1bcf8e0d72f15e710ff0c03"), - Linux(:armv7l, :musl, :eabihf) => ("$bin_prefix/IConv.arm-linux-musleabihf.tar.gz", "200db91435875f57ea827ef69b31b6d4c11f089aacc695846699155488245b1d"), - Linux(:i686, :glibc) => ("$bin_prefix/IConv.i686-linux-gnu.tar.gz", "c0dea5b3c6ebfc0ae0d3b3c1affe25a93bb11c5da8dd34c2a40ffb132e61e4ee"), - Linux(:i686, :musl) => ("$bin_prefix/IConv.i686-linux-musl.tar.gz", "96cccd8a539c3ad4a70ddade4396f91015248d6fad799adcf0f79f3940d1c672"), - Windows(:i686) => ("$bin_prefix/IConv.i686-w64-mingw32.tar.gz", "7d8b600337c4153f9a4ce2115f071aabc9a8c5565cb95cad4c6cc3dd6df17fb3"), - Linux(:powerpc64le, :glibc) => ("$bin_prefix/IConv.powerpc64le-linux-gnu.tar.gz", "8e7edc2adbd331cb1f0a944f7eb26ed56ac884830d4d65785671c0e620be84e6"), - MacOS(:x86_64) => ("$bin_prefix/IConv.x86_64-apple-darwin14.tar.gz", "5b585a372c272aed10823f5ea4473849396ee928f9f6b7643411138549349bf2"), - Linux(:x86_64, :glibc) => ("$bin_prefix/IConv.x86_64-linux-gnu.tar.gz", "6f58cd9722f492aa0e519c9d91082c0b0776bd53cbff17f643172fec2c4d4347"), - Linux(:x86_64, :musl) => ("$bin_prefix/IConv.x86_64-linux-musl.tar.gz", "c72aaabc45465fdb2b7d74df2bd4a0282cf1d2a46e55868b129003547396b69f"), - FreeBSD(:x86_64) => ("$bin_prefix/IConv.x86_64-unknown-freebsd11.1.tar.gz", "69a5585871a61547e71cd06d32269e972f2d7ab488f82ffb852b3b5beaa83d8f"), - Windows(:x86_64) => ("$bin_prefix/IConv.x86_64-w64-mingw32.tar.gz", "c553d5d36ad831c13572b1c06230b41471b1dea560fbc03d5213cf4304de74cf"), -) - -# Detect already present libc iconv or libiconv -# (notably for Linux, Mac OS and other Unixes) -global found_iconv = false -for lib in ("libc", "libc-bin", "libiconv", "iconv") - if lib in ("libc", "libc-bin") - global iconv_open = :iconv_open - global iconv_close = :iconv_close - global iconv = :iconv - else - global iconv_open = :libiconv_open - global iconv_close = :libiconv_close - global iconv = :libiconv - end - if validate_iconv(lib, iconv_open, iconv_close, iconv) - global found_iconv = true - write(joinpath(@__DIR__, "deps.jl"), - """ - ## This file autogenerated by Pkg.build(\\"StringEncodings\\"). - ## Do not edit. - const libiconv = "$lib" - function check_deps() - global libiconv - if Libdl.dlopen_e(libiconv) == C_NULL - error("\$(libiconv) cannot be opened. Please re-run Pkg.build(\\"StringEncodings\\"), and restart Julia.") - end - - end - """) - break - end -end - -if !found_iconv - iconv_open = :libiconv_open - iconv_close = :libiconv_close - iconv = :libiconv - # Install unsatisfied or updated dependencies: - unsatisfied = any(!satisfied(p; verbose=verbose) for p in products) - if haskey(download_info, platform_key()) - url, tarball_hash = download_info[platform_key()] - if unsatisfied || !isinstalled(url, tarball_hash; prefix=prefix) - # Download and install binaries - install(url, tarball_hash; prefix=prefix, force=true, verbose=verbose) - end - elseif unsatisfied - # If we don't have a BinaryProvider-compatible .tar.gz to download, complain. - # Alternatively, you could attempt to install from a separate provider, - # build from source or something even more ambitious here. - error("Your platform $(triplet(platform_key())) is not supported by this package! Try installing libiconv manually.") - end - - # Write out a deps.jl file that will contain mappings for our products - write_deps_file(joinpath(@__DIR__, "deps.jl"), products) -end - -open(joinpath(@__DIR__, "deps.jl"), "a") do io - write(io, - """ - const iconv_open_s = :$iconv_open - const iconv_close_s = :$iconv_close - const iconv_s = :$iconv - """) -end diff --git a/src/StringEncodings.jl b/src/StringEncodings.jl index 3714f74..7d780d3 100644 --- a/src/StringEncodings.jl +++ b/src/StringEncodings.jl @@ -2,20 +2,7 @@ module StringEncodings -using Libdl - -# Load in `deps.jl`, complaining if it does not exist -const depsjl_path = joinpath(dirname(@__FILE__), "..", "deps", "deps.jl") -if !isfile(depsjl_path) - error("iconv not installed properly, run Pkg.build(\"StringEncodings\"), restart Julia and try again") -end -include(depsjl_path) - -# Module initialization function -function __init__() - # Always check dependencies from `deps.jl` - check_deps() -end +using Libiconv_jll using Base.Libc: errno, strerror, E2BIG, EINVAL, EILSEQ @@ -75,13 +62,13 @@ show(io::IO, exc::T) where {T<:Union{IncompleteSequenceError,OutputBufferError}} function iconv_close(cd::Ptr{Nothing}) if cd != C_NULL - ccall((iconv_close_s, libiconv), Cint, (Ptr{Nothing},), cd) == 0 || + ccall((:libiconv_close, libiconv), Cint, (Ptr{Nothing},), cd) == 0 || throw(IConvError("iconv_close")) end end function iconv_open(tocode::String, fromcode::String) - p = ccall((iconv_open_s, libiconv), Ptr{Nothing}, (Cstring, Cstring), tocode, fromcode) + p = ccall((:libiconv_open, libiconv), Ptr{Nothing}, (Cstring, Cstring), tocode, fromcode) if p != Ptr{Nothing}(-1) return p elseif errno() == EINVAL @@ -144,7 +131,7 @@ function iconv!(cd::Ptr{Nothing}, inbuf::Vector{UInt8}, outbuf::Vector{UInt8}, inbytesleft_orig = inbytesleft[] outbytesleft[] = BUFSIZE - ret = ccall((iconv_s, libiconv), Csize_t, + ret = ccall((:libiconv, libiconv), Csize_t, (Ptr{Nothing}, Ptr{Ptr{UInt8}}, Ref{Csize_t}, Ptr{Ptr{UInt8}}, Ref{Csize_t}), cd, inbufptr, inbytesleft, outbufptr, outbytesleft) @@ -176,7 +163,7 @@ function iconv_reset!(s::Union{StringEncoder, StringDecoder}) s.outbufptr[] = pointer(s.outbuf) s.outbytesleft[] = BUFSIZE - ret = ccall((iconv_s, libiconv), Csize_t, + ret = ccall((:libiconv, libiconv), Csize_t, (Ptr{Nothing}, Ptr{Ptr{UInt8}}, Ref{Csize_t}, Ptr{Ptr{UInt8}}, Ref{Csize_t}), s.cd, C_NULL, C_NULL, s.outbufptr, s.outbytesleft) @@ -278,9 +265,6 @@ read from `stream` into text in the encoding `to`. Calling `close` on the stream does not close `stream`. `to` and `from` can be specified either as a string or as an `Encoding` object. - -Note that some implementations may accept invalid sequences -in the input data without raising an error. """ function StringDecoder(stream::IO, from::Encoding, to::Encoding=enc"UTF-8") cd = iconv_open(String(to), String(from)) @@ -542,7 +526,7 @@ encode(s::AbstractString, enc::AbstractString) = encode(s, Encoding(enc)) function test_encoding(enc::String) # We assume that an encoding is supported if it's possible to convert from it to UTF-8: - cd = ccall((iconv_open_s, libiconv), Ptr{Nothing}, (Cstring, Cstring), enc, "UTF-8") + cd = ccall((:libiconv_open, libiconv), Ptr{Nothing}, (Cstring, Cstring), enc, "UTF-8") if cd == Ptr{Nothing}(-1) return false else From a250678c177e52235c079d047f2b7a2f6c033582 Mon Sep 17 00:00:00 2001 From: Alex Arslan Date: Fri, 21 May 2021 12:05:43 -0700 Subject: [PATCH 3/4] Bump patch version --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index e8cc82b..83c9c49 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "StringEncodings" uuid = "69024149-9ee7-55f6-a4c4-859efe599b68" -version = "0.3.3" +version = "0.3.4" [deps] Libiconv_jll = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" From d74c52eb726758803eb5dd69c8c6d3eeb02d8ba3 Mon Sep 17 00:00:00 2001 From: Milan Bouchet-Valat Date: Sun, 23 May 2021 23:08:57 +0200 Subject: [PATCH 4/4] Update docstrings --- src/StringEncodings.jl | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/StringEncodings.jl b/src/StringEncodings.jl index 7d780d3..55e950d 100644 --- a/src/StringEncodings.jl +++ b/src/StringEncodings.jl @@ -539,15 +539,13 @@ end encodings() List all encodings supported by `encode`, `decode`, `StringEncoder` and `StringDecoder` -(i.e. by the current iconv implementation). +(i.e. by GNU libiconv). Note that encodings typically appear several times under different names. In addition to the encodings returned by this function, the empty string (i.e. `""`) is equivalent to the encoding of the current locale. -Some implementations may support even more encodings: this can be checked by attempting -a conversion. In theory, it is not guaranteed that all conversions between all pairs of encodings -are possible; but this is the case with all reasonable implementations. +Even more encodings may be supported: this can be checked by attempting a conversion. """ function encodings() filter(test_encoding, encodings_list)