From 8295d4b67f7d461a991fa34507b0e2c74a3201ea Mon Sep 17 00:00:00 2001 From: Rudolf M Date: Fri, 1 May 2020 01:24:44 -0700 Subject: [PATCH] add hex_tarball:unpack/3 that accepts list of files to unpack --- README.md | 8 ++++++++ src/hex_tarball.erl | 39 +++++++++++++++++++++++++++----------- test/hex_tarball_SUITE.erl | 20 ++++++++++++++++++- 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 4afbef7..e52a4ec 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,14 @@ Unpack package tarball: {ok, #{outer_checksum := Checksum, contents := Contents, metadata := Metadata}} = hex_tarball:unpack(Tarball, memory). ``` +Or provide the list of files you want to extract from the tarball + +```erlang +File = "dir/foo", +{ok, #{outer_checksum := Checksum, contents := [{File, Content}], metadata := Metadata}} = hex_tarball:unpack(Tarball, [File], memory). +``` + + Remember to verify the outer tarball checksum against the registry checksum returned from `hex_repo:get_package(Config, Package)`. diff --git a/src/hex_tarball.erl b/src/hex_tarball.erl index 910e7e1..b697461 100644 --- a/src/hex_tarball.erl +++ b/src/hex_tarball.erl @@ -1,5 +1,5 @@ -module(hex_tarball). --export([create/2, create_docs/1, unpack/2, unpack_docs/2, format_checksum/1, format_error/1]). +-export([create/2, create_docs/1, unpack/2, unpack/3, unpack_docs/2, format_checksum/1, format_error/1]). -ifdef(TEST). -export([do_decode_metadata/1, gzip/1, normalize_requirements/1]). -endif. @@ -112,6 +112,12 @@ create_docs(Files) -> %% contents => [{"src/foo.erl",<<"-module(foo).">>}], %% metadata => #{<<"name">> => <<"foo">>, ...}}} %% +%% > hex_tarball:unpack(Tarball, ["src/foo", "src/bar"], memory). +%% {ok,#{outer_checksum => <<...>>, +%% contents => [{"src/foo",<<...>>}, +%% {"src/bar",<<...>>}], +%% metadata => #{<<"name">> => <<"foo">>, ...}}} +%% %% > hex_tarball:unpack(Tarball, "path/to/unpack"). %% {ok,#{outer_checksum => <<...>>, %% metadata => #{<<"name">> => <<"foo">>, ...}}} @@ -128,13 +134,20 @@ unpack(Tarball, _) when byte_size(Tarball) > ?TARBALL_MAX_SIZE -> {error, {tarball, too_big}}; unpack(Tarball, Output) -> + unpack(Tarball, [], Output). + +unpack(Tarball, ListOfFilesToExtract, Output) -> case hex_erl_tar:extract({binary, Tarball}, [memory]) of {ok, []} -> {error, {tarball, empty}}; {ok, FileList} -> OuterChecksum = crypto:hash(sha256, Tarball), - do_unpack(maps:from_list(FileList), OuterChecksum, Output); + Opts = case ListOfFilesToExtract of + [] -> []; + _ -> [{files, ListOfFilesToExtract}] + end, + do_unpack(maps:from_list(FileList), OuterChecksum, Output, Opts); {error, Reason} -> {error, {tarball, Reason}} @@ -206,7 +219,7 @@ encode_metadata(Meta) -> end, maps:to_list(Meta)), iolist_to_binary(Data). -do_unpack(Files, OuterChecksum, Output) -> +do_unpack(Files, OuterChecksum, Output, Opts) -> State = #{ inner_checksum => undefined, outer_checksum => OuterChecksum, @@ -219,15 +232,16 @@ do_unpack(Files, OuterChecksum, Output) -> State2 = check_version(State1), State3 = check_inner_checksum(State2), State4 = decode_metadata(State3), - finish_unpack(State4). + finish_unpack(State4, Opts). -finish_unpack({error, _} = Error) -> +finish_unpack({error, _} = Error, _) -> Error; -finish_unpack(#{metadata := Metadata, files := Files, inner_checksum := InnerChecksum, outer_checksum := OuterChecksum, output := Output}) -> + +finish_unpack(#{metadata := Metadata, files := Files, inner_checksum := InnerChecksum, outer_checksum := OuterChecksum, output := Output}, Opts) -> _ = maps:get("VERSION", Files), ContentsBinary = maps:get("contents.tar.gz", Files), filelib:ensure_dir(filename:join(Output, "*")), - case unpack_tarball(ContentsBinary, Output) of + case unpack_tarball(ContentsBinary, Output, Opts) of ok -> copy_metadata_config(Output, maps:get("metadata.config", Files)), {ok, #{inner_checksum => InnerChecksum, outer_checksum => OuterChecksum, metadata => Metadata}}; @@ -360,12 +374,15 @@ guess_build_tools(Metadata) -> %%==================================================================== %% Tar Helpers %%==================================================================== - -unpack_tarball(ContentsBinary, memory) -> - hex_erl_tar:extract({binary, ContentsBinary}, [memory, compressed]); unpack_tarball(ContentsBinary, Output) -> + unpack_tarball(ContentsBinary, Output, []). + +unpack_tarball(ContentsBinary, memory, Opts) -> + hex_erl_tar:extract({binary, ContentsBinary}, [memory, compressed | Opts]); + +unpack_tarball(ContentsBinary, Output, Opts) -> filelib:ensure_dir(filename:join(Output, "*")), - case hex_erl_tar:extract({binary, ContentsBinary}, [{cwd, Output}, compressed]) of + case hex_erl_tar:extract({binary, ContentsBinary}, [{cwd, Output}, compressed | Opts]) of ok -> [try_updating_mtime(filename:join(Output, Path)) || Path <- filelib:wildcard("**", Output)], ok; diff --git a/test/hex_tarball_SUITE.erl b/test/hex_tarball_SUITE.erl index 6d74eab..f929a97 100644 --- a/test/hex_tarball_SUITE.erl +++ b/test/hex_tarball_SUITE.erl @@ -10,7 +10,7 @@ all() -> [disk_test, timestamps_and_permissions_test, symlinks_test, memory_test, build_tools_test, requirements_test, decode_metadata_test, unpack_error_handling_test, - docs_test + docs_test, unpack_all_files_test, unpack_list_of_files_test ]. memory_test(_Config) -> @@ -241,6 +241,24 @@ docs_test(Config) -> ok. +unpack_all_files_test(_Config) -> + Metadata = #{<<"name">> => <<"foo">>, <<"version">> => <<"1.0.0">>}, + Files = [{"foo", <<"">>},{"bar", <<"">>}], + {ok, #{tarball := Tarball}} = hex_tarball:create(Metadata, Files), + + {ok, #{contents := Files}} = hex_tarball:unpack(Tarball, memory), + + ok. + +unpack_list_of_files_test(_Config) -> + Metadata = #{<<"name">> => <<"foo">>, <<"version">> => <<"1.0.0">>}, + ExpectedFile = {"foo", <<"">>}, + {ok, #{tarball := Tarball}} = hex_tarball:create(Metadata, [ExpectedFile, {"bar", <<"">>}]), + + {ok, #{contents := [ExpectedFile]}} = hex_tarball:unpack(Tarball, ["foo"], memory), + + ok. + %%==================================================================== %% Helpers %%====================================================================