Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
dbf8861
Patching pre/post install/remove scripts from RPM task to packager eq…
Aug 11, 2018
ebc9045
updated to current (8 years old) file color scheme and classifying .N…
Aug 11, 2018
cfc1ea5
read metadata from RPM instead of guessing it. introducing Documentat…
Aug 12, 2018
f13682a
fix parameter bug in test RpmPackageCreatorTests.CreatePackageTest
Aug 12, 2018
8cb6b26
Merge pull request #63 from clemensv/issue62
qmfrederik Aug 13, 2018
1e758f1
Update Packaging.Targets.targets
darkneil14 Aug 30, 2018
fdc35b3
Merge pull request #66 from darkneil14/master
qmfrederik Aug 30, 2018
d80ddbd
Use NerdBank.GitVersioning to assign build numbers
qmfrederik Aug 30, 2018
693c8fa
Update AppVeyor file to use NB.GV-produced assets
qmfrederik Aug 30, 2018
fc8c949
Don't start branch builds if a PR is active
qmfrederik Aug 30, 2018
b2b0de7
Merge pull request #67 from qmfrederik/fixes/nb.gv
qmfrederik Aug 30, 2018
c19298e
Merge pull request #68 from qmfrederik/skip-branch-with-pr
qmfrederik Aug 30, 2018
333273c
Add Travis CI build (#69)
qmfrederik Aug 30, 2018
85e3bb2
Release from master
qmfrederik Aug 31, 2018
614bdbf
Move the demo project to demo/aspnetcore, so we can have multiple dem…
qmfrederik Aug 31, 2018
114f34c
Use the latest version of dotnet-packaging in the demo projects
qmfrederik Aug 31, 2018
bba3266
Use Release config
qmfrederik Aug 31, 2018
667f91d
Run dotnet deb, dotnet rpm as part of the build
qmfrederik Aug 31, 2018
5214cc2
Merge pull request #72 from qmfrederik/fixes/ci-run
qmfrederik Aug 31, 2018
85cc823
Merge pull request #71 from qmfrederik/fixes/release-master
qmfrederik Aug 31, 2018
2f06734
Bump libicu versions to add support for Ubuntu 18.04
qmfrederik Aug 31, 2018
34844e8
Merge pull request #73 from qmfrederik/fixes/ubuntu-18.04
qmfrederik Aug 31, 2018
2a612e2
docker test updates
Aug 28, 2018
2030f61
docker tests
Aug 29, 2018
7e9d088
only build the images branch from root
Aug 29, 2018
4de2cf5
call tests
Aug 29, 2018
a0f9d7b
Move demo projects to demo/
qmfrederik Aug 31, 2018
c27b07f
Install Debian 8 package in a Docker container
qmfrederik Aug 31, 2018
375b24d
Run tests on CentOS 7
qmfrederik Aug 31, 2018
11271fa
Disable Docker on AppVeyor as Linux containers on Windows are availab…
qmfrederik Aug 31, 2018
a0bef75
Add tests for Ubuntu 18.04
qmfrederik Aug 31, 2018
2d9e358
Add Ubuntu 16.04
qmfrederik Aug 31, 2018
b2d9526
Add OpenSUSE 42.3
qmfrederik Aug 31, 2018
c73b8b2
Add Oracle Linux & Fedora tests
qmfrederik Aug 31, 2018
a482d7e
Remove test/ directory, everything has been moved to demo/
qmfrederik Aug 31, 2018
76d933e
Don't run on OpenSUSE for nwo
qmfrederik Aug 31, 2018
0bffc23
fix typo
qmfrederik Aug 31, 2018
d74fdbb
Merge pull request #70 from qmfrederik/dockertest-2
qmfrederik Aug 31, 2018
c47ecd0
Only mark files in "/usr/share/doc" as documentation files
qmfrederik Aug 31, 2018
5e80a6f
Update Docker files with fixes for RPM packages
qmfrederik Aug 31, 2018
03a32b1
Skip distros for which we don't specify libicu dependencies
qmfrederik Aug 31, 2018
15a0059
Merge pull request #74 from qmfrederik/fixes/rpm-doc-files
qmfrederik Aug 31, 2018
12207f6
Expand CI coverage
qmfrederik Aug 31, 2018
825031a
Require sudo
qmfrederik Sep 1, 2018
92133ef
Use parallel builds
qmfrederik Sep 1, 2018
8ed5793
Try to speed up builds
qmfrederik Sep 1, 2018
e32cbd4
Fix installation on OpenSUSE
qmfrederik Sep 1, 2018
d737c44
Merge pull request #75 from qmfrederik/fixes/ci-coverage
qmfrederik Sep 1, 2018
693e999
updated to current (8 years old) file color scheme and classifying .N…
Aug 11, 2018
dfbd117
read metadata from RPM instead of guessing it. introducing Documentat…
Aug 12, 2018
1c84878
Merge branch 'issue61' of https://github.com/clemensv/dotnet-packagin…
Sep 5, 2018
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{
// Use IntelliSense to find out which attributes exist for C# debugging
// Use hover for the description of the existing attributes
// For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md
"version": "0.2.0",
"configurations": [
{
"name": ".NET Core Launch (console)",
"type": "coreclr",
"request": "launch",
"preLaunchTask": "build",
// If you have changed target frameworks, make sure to update the program path.
"program": "${workspaceFolder}/dotnet-deb/bin/Debug/netcoreapp1.0/dotnet-deb.dll",
"args": [],
"cwd": "${workspaceFolder}/dotnet-deb",
// For more information about the 'console' field, see https://github.com/OmniSharp/omnisharp-vscode/blob/master/debugger-launchjson.md#console-terminal-window
"console": "internalConsole",
"stopAtEntry": false,
"internalConsoleOptions": "openOnSessionStart"
},
{
"name": ".NET Core Attach",
"type": "coreclr",
"request": "attach",
"processId": "${command:pickProcess}"
}
,]
}
15 changes: 15 additions & 0 deletions .vscode/tasks.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{
"version": "2.0.0",
"tasks": [
{
"label": "build",
"command": "dotnet",
"type": "process",
"args": [
"build",
"${workspaceFolder}/dotnet-deb/dotnet-deb.csproj"
],
"problemMatcher": "$msCompile"
}
]
}
4 changes: 2 additions & 2 deletions Packaging.Targets.Tests/Rpm/RpmMetadataTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ public void SetFilesTest()
{
ArchiveBuilder builder = new ArchiveBuilder(new PlistFileAnalyzer());
RpmPackageCreator creator = new RpmPackageCreator(new PlistFileAnalyzer());
var entries = builder.FromCpio(cpio);
var entries = builder.FromCpio(originalPackage, cpio);
var files = creator.CreateFiles(entries);

var metadata = new PublicRpmMetadata(package);
Expand Down Expand Up @@ -186,7 +186,7 @@ public void CreatePackageMetadata()
{
ArchiveBuilder builder = new ArchiveBuilder(new PlistFileAnalyzer());
RpmPackageCreator creator = new RpmPackageCreator(new PlistFileAnalyzer());
var entries = builder.FromCpio(cpio);
var entries = builder.FromCpio(originalPackage, cpio);
var files = creator.CreateFiles(entries);

// Core routine to populate files and dependencies
Expand Down
10 changes: 5 additions & 5 deletions Packaging.Targets.Tests/Rpm/RpmPackageCreatorTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public void CreateFiles(string rpm, string analyzerName)

RpmPackageCreator creator = new RpmPackageCreator(analyzer);
ArchiveBuilder builder = new ArchiveBuilder(analyzer);
var entries = builder.FromCpio(cpio);
var entries = builder.FromCpio(originalPackage, cpio);
var files = creator.CreateFiles(entries);

var originalMetadata = new RpmMetadata(originalPackage);
Expand Down Expand Up @@ -98,7 +98,7 @@ public void CalculateOffsetTest()
{
ArchiveBuilder builder = new ArchiveBuilder(new PlistFileAnalyzer());
RpmPackageCreator creator = new RpmPackageCreator(new PlistFileAnalyzer());
var entries = builder.FromCpio(cpio);
var entries = builder.FromCpio(originalPackage, cpio);
var files = creator.CreateFiles(entries);

// Core routine to populate files and dependencies
Expand Down Expand Up @@ -151,7 +151,7 @@ public void CalculateSignatureTest()
using (var cpio = new CpioFile(payloadStream, false))
{
ArchiveBuilder builder = new ArchiveBuilder(new PlistFileAnalyzer());
var entries = builder.FromCpio(cpio);
var entries = builder.FromCpio(originalPackage, cpio);
files = creator.CreateFiles(entries);
}

Expand Down Expand Up @@ -253,7 +253,7 @@ public void CreatePackageTest()
using (CpioFile cpio = new CpioFile(decompressedPayloadStream, leaveOpen: false))
{
ArchiveBuilder builder = new ArchiveBuilder();
archive = builder.FromCpio(cpio);
archive = builder.FromCpio(originalPackage, cpio);
}

using (var decompressedPayloadStream = RpmPayloadReader.GetDecompressedPayloadStream(originalPackage))
Expand Down Expand Up @@ -318,7 +318,7 @@ public void CreatePackageBinaryTest()
using (CpioFile cpio = new CpioFile(decompressedPayloadStream, leaveOpen: false))
{
ArchiveBuilder builder = new ArchiveBuilder();
archive = builder.FromCpio(cpio);
archive = builder.FromCpio(originalPackage, cpio);
}

using (var compressedPayloadStream = RpmPayloadReader.GetCompressedPayloadStream(originalPackage))
Expand Down
103 changes: 85 additions & 18 deletions Packaging.Targets/ArchiveBuilder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
using System.Collections.ObjectModel;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Runtime.Loader;
using System.Security.Cryptography;
using System.Text;

Expand Down Expand Up @@ -46,20 +48,23 @@ public ArchiveBuilder(IFileAnalyzer analyzer)
/// <summary>
/// Extracts the <see cref="ArchiveEntry"/> objects from a CPIO file.
/// </summary>
/// <param name="rpmPackage"></param>
/// <param name="file">
/// The CPIO file from which to extract the entries.
/// The CPIO file from which to extract the entries.
/// </param>
/// <returns>
/// A list of <see cref="ArchiveEntry"/> objects representing the data in the CPIO file.
/// </returns>
public List<ArchiveEntry> FromCpio(CpioFile file)
public List<ArchiveEntry> FromCpio(RpmPackage rpmPackage, CpioFile file)
{
List<ArchiveEntry> value = new List<ArchiveEntry>();
byte[] buffer = new byte[1024];
byte[] fileHeader = null;
int currentEntry = -1;

while (file.Read())
{
currentEntry++;
fileHeader = null;

ArchiveEntry entry = new ArchiveEntry()
Expand All @@ -75,9 +80,40 @@ public List<ArchiveEntry> FromCpio(CpioFile file)
LinkTo = string.Empty,
Sha256 = Array.Empty<byte>(),
SourceFilename = null,
IsAscii = true
IsAscii = true,

};


var fileColor = (RpmFileColor)(rpmPackage.Header.Records[IndexTag.RPMTAG_FILECOLORS]?.Value as Collection<int>)?[currentEntry];
var fileFlag = (RpmFileFlags)(rpmPackage.Header.Records[IndexTag.RPMTAG_FILEFLAGS]?.Value as Collection<int>)?[currentEntry];
var fileClassIndex = (int)(rpmPackage.Header.Records[IndexTag.RPMTAG_FILECLASS]?.Value as Collection<int>)?[currentEntry];
var fileClass = (rpmPackage.Header.Records[IndexTag.RPMTAG_CLASSDICT]?.Value as Collection<string>)?[fileClassIndex];

entry.Group = (rpmPackage.Header.Records[IndexTag.RPMTAG_FILEGROUPNAME]?.Value as Collection<string>)?[currentEntry];
entry.Owner = (rpmPackage.Header.Records[IndexTag.RPMTAG_FILEUSERNAME]?.Value as Collection<string>)?[currentEntry];
entry.LinkTo = (rpmPackage.Header.Records[IndexTag.RPMTAG_FILELINKTOS]?.Value as Collection<string>)?[currentEntry];

if ((fileFlag & RpmFileFlags.RPMFILE_DOC) == RpmFileFlags.RPMFILE_DOC)
{
entry.Type = ArchiveEntryType.Doc;
}

if ((fileColor & RpmFileColor.RPMFC_ELF32) == RpmFileColor.RPMFC_ELF32)
{
entry.Type = ArchiveEntryType.Executable32;
}

if ((fileColor & RpmFileColor.RPMFC_ELF64) == RpmFileColor.RPMFC_ELF64)
{
entry.Type = ArchiveEntryType.Executable64;
}

if (fileClass.Contains("mono"))
{
entry.Type = ArchiveEntryType.NetAssembly;
}

if (entry.Mode.HasFlag(LinuxFileMode.S_IFREG) && !entry.Mode.HasFlag(LinuxFileMode.S_IFLNK))
{
using (var fileStream = file.Open())
Expand Down Expand Up @@ -106,8 +142,6 @@ public List<ArchiveEntry> FromCpio(CpioFile file)

entry.Sha256 = hasher.GetHashAndReset();
}

entry.Type = this.GetArchiveEntryType(fileHeader);
}
else if (entry.Mode.HasFlag(LinuxFileMode.S_IFLNK))
{
Expand Down Expand Up @@ -277,25 +311,58 @@ protected void AddFile(string entry, string relativePath, string prefix, List<Ar
md5hash = md5hasher.GetHashAndReset();
}

// Only support ELF32 and ELF64 colors; otherwise default to BLACK.
ArchiveEntryType entryType = this.GetArchiveEntryType(fileHeader);

var mode = LinuxFileMode.S_IROTH | LinuxFileMode.S_IRGRP | LinuxFileMode.S_IRUSR | LinuxFileMode.S_IFREG;

if (entryType == ArchiveEntryType.Executable32 || entryType == ArchiveEntryType.Executable64)
ArchiveEntryType entryType;
if (fileMetadata != null && fileMetadata.GetDocumentation())
{
mode |= LinuxFileMode.S_IXOTH | LinuxFileMode.S_IXGRP | LinuxFileMode.S_IWUSR | LinuxFileMode.S_IXUSR;
entryType = ArchiveEntryType.Doc;
}
else
{
// check whether it's an ELF file
entryType = this.GetArchiveEntryType(fileHeader);

// If a Linux path has been specified, use that one, else, use the default one based on the prefix
// + current file name.
string name = fileMetadata?.GetLinuxPath();
// check whether this is a .NET assembly
if (entryType == ArchiveEntryType.None && File.Exists(fileName))
{
try
{
var assemblyName = AssemblyLoadContext.GetAssemblyName(fileName);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This try/catch sounds very expensive, and we can probably apply some more logic to determine whether a file is a .NET executable.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Exceptions are worrisome if they're on the hot path for performance sensitive code that runs on a sustained basis, like inside an ASP.NET app. This here is done once for each file during packaging of an app, so I don't find that of concern. The alternative is to pull in PE file decoder code, which is far more code (I started with that approach and discarded it). We could safeguard the section with a simple probe for the 'MZ' magic word being the first two bytes of the file content which should catch most cases.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could safeguard the section with a simple probe for the 'MZ' magic word being the first two bytes of the file content which should catch most cases.

That sounds like a good compromise.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll add that.

entryType = ArchiveEntryType.NetAssembly;
}
catch (BadImageFormatException)
{
}
}
}

if (name == null)
LinuxFileMode mode = 0;
if (fileMetadata.GetLinuxFileMode() != null)
{
// allow for the file mode to use numbers or symbols per
// our enum and bitwise or
var smodes = fileMetadata.GetLinuxFileMode().Split('|', ' ');
foreach (var s in smodes)
{
if (Enum.TryParse(s, out LinuxFileMode m))
{
mode |= m;
}
}
}
else
{
name = prefix + "/" + fileName;
mode = LinuxFileMode.S_IROTH | LinuxFileMode.S_IRGRP | LinuxFileMode.S_IRUSR |
LinuxFileMode.S_IFREG;
if (entryType == ArchiveEntryType.Executable32 || entryType == ArchiveEntryType.Executable64)
{
mode |= LinuxFileMode.S_IXOTH | LinuxFileMode.S_IXGRP | LinuxFileMode.S_IWUSR |
LinuxFileMode.S_IXUSR;
}
}

// If a Linux path has been specified, use that one, else, use the default one based on the prefix
// + current file name.
string name = fileMetadata?.GetLinuxPath() ?? prefix + "/" + fileName;
string linkTo = string.Empty;

if (mode.HasFlag(LinuxFileMode.S_IFLNK))
Expand All @@ -322,7 +389,7 @@ protected void AddFile(string entry, string relativePath, string prefix, List<Ar
{
mode = (LinuxFileMode)Convert.ToUInt32(overridenFileMode, 8);
}
catch (Exception ex)
catch (Exception)
{
throw new Exception($"Could not parse the file mode '{overridenFileMode}' for file '{name}'. Make sure to set the LinuxFileMode attriubute to an octal representation of a Unix file mode.");
}
Expand Down
12 changes: 11 additions & 1 deletion Packaging.Targets/IO/ArchiveEntryType.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,16 @@ public enum ArchiveEntryType
/// <summary>
/// The file is a 64-bit executable.
/// </summary>
Executable64 = 2
Executable64 = 2,

/// <summary>
/// The file is a module (like a .NET assembly)
/// </summary>
NetAssembly = 3,

/// <summary>
/// This file is a documentation entry
/// </summary>
Doc = 4
}
}
1 change: 1 addition & 0 deletions Packaging.Targets/Packaging.Targets.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
<PackageReference Include="SharpZipLib.NETStandard" Version="0.86.0.1">
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="System.Runtime.Loader" Version="4.3.0" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFramework)' == 'netstandard1.5'">
<PackageReference Include="System.Globalization.Extensions" Version="4.3.0" />
Expand Down
22 changes: 9 additions & 13 deletions Packaging.Targets/Rpm/FileAnalyzer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -33,19 +33,7 @@ public virtual Collection<PackageDependency> DetermineProvides(ArchiveEntry entr
/// <inheritdoc/>
public virtual RpmFileFlags DetermineFlags(ArchiveEntry entry)
{
// The only custom flags which are supported for now are the RPMFILE_DOC flags for non-executable
// files.
if (entry.Mode.HasFlag(LinuxFileMode.S_IFDIR))
{
return RpmFileFlags.None;
}
else if (entry.Mode.HasFlag(LinuxFileMode.S_IFLNK))
{
return RpmFileFlags.None;
}
else if (!entry.Mode.HasFlag(LinuxFileMode.S_IXGRP)
&& !entry.Mode.HasFlag(LinuxFileMode.S_IXOTH)
&& !entry.Mode.HasFlag(LinuxFileMode.S_IXUSR))
if (entry.Type == ArchiveEntryType.Doc)
{
return RpmFileFlags.RPMFILE_DOC;
}
Expand All @@ -67,6 +55,9 @@ public virtual RpmFileColor DetermineColor(ArchiveEntry entry)
case ArchiveEntryType.Executable64:
return RpmFileColor.RPMFC_ELF64;

case ArchiveEntryType.NetAssembly:
return RpmFileColor.RPMFC_INCLUDE;

default:
return RpmFileColor.RPMFC_BLACK;
}
Expand All @@ -92,6 +83,11 @@ public virtual string DetermineClass(ArchiveEntry entry)
return string.Empty;
}

if (entry.Type == ArchiveEntryType.NetAssembly)
{
return "mono";
}

if (entry.TargetPath.EndsWith(".svg"))
{
return "SVG Scalable Vector Graphics image";
Expand Down
32 changes: 6 additions & 26 deletions Packaging.Targets/Rpm/RpmFileColor.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,41 +2,21 @@

namespace Packaging.Targets.Rpm
{
/*
* from https://github.com/rpm-software-management/rpm/blob/master/build/rpmfc.h
*/

/// <summary>
/// Determines the type of the file.
/// </summary>
[Flags]
internal enum RpmFileColor
internal enum RpmFileColor : int
{
RPMFC_BLACK = 0,
RPMFC_ELF32 = 1 << 0,
RPMFC_ELF64 = 1 << 1,
RPMFC_ELFMIPSN32 = 1 << 2,
RPMFC_PKGCONFIG = 1 << 4,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we keep these enum values? They map to the values defined in the RPM API I would prefer to keep them in sync.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did some research before I pulled those values out.

The values you have here got dropped from RPM in 2010 and RPM's classification model doesn't use them anymore but rather uses the text classifier dictionary. My edits align to the current rpmfc.h header.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, sounds good then!

RPMFC_LIBTOOL = 1 << 5,
RPMFC_BOURNE = 1 << 6,
RPMFC_MODULE = 1 << 7,
RPMFC_EXECUTABLE = 1 << 8,
RPMFC_SCRIPT = 1 << 9,
RPMFC_TEXT = 1 << 10,
RPMFC_DATA = 1 << 11,
RPMFC_DOCUMENT = 1 << 12,
RPMFC_STATIC = 1 << 13,
RPMFC_NOTSTRIPPED = 1 << 14,
RPMFC_COMPRESSED = 1 << 15,
RPMFC_DIRECTORY = 1 << 16,
RPMFC_SYMLINK = 1 << 17,
RPMFC_DEVICE = 1 << 18,
RPMFC_LIBRARY = 1 << 19,
RPMFC_ARCHIVE = 1 << 20,
RPMFC_FONT = 1 << 21,
RPMFC_IMAGE = 1 << 22,
RPMFC_MANPAGE = 1 << 23,
RPMFC_PERL = 1 << 24,
RPMFC_JAVA = 1 << 25,
RPMFC_PYTHON = 1 << 26,
RPMFC_PHP = 1 << 27,
RPMFC_TCL = 1 << 28,
RPMFC_ELF = RPMFC_ELF32 | RPMFC_ELF64 | RPMFC_ELFMIPSN32,
RPMFC_WHITE = 1 << 29,
RPMFC_INCLUDE = 1 << 30,
RPMFC_ERROR = 1 << 31
Expand Down
Loading