1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
|
{
lib,
config,
pkgs,
...
}:
let
Dependency =
{ name, ... }:
{
options.name = lib.mkOption {
type = lib.types.str;
default = name;
};
options.runtime = lib.mkEnableOption "Keep a reference in the output store path to retain a runtime dependency";
};
Node = (
{ name, ... }:
{
options.name = lib.mkOption {
type = lib.types.str;
default = name;
};
options.request = lib.mkEnableOption "Whether to mark the node for building";
options.needed = lib.mkOption {
type = with lib.types; nullOr bool;
default = null;
description = "Whether the node must be built to satisfy all requests (either a requested node or a transitive dependency)";
};
options.chosen = lib.mkOption {
type = with lib.types; nullOr bool;
default = null;
description = "Whether the node is included in the build plan (i.t. it's `needed` and fitted into budget)";
};
options.cache = lib.mkOption {
type = lib.types.enum [
"none"
"remote"
"local"
];
description = ''
Whether the dependency is pre-built and available in the local /nix/store ("local"), can be substituted ("remote"), or has to be built ("none")
'';
default = "none";
};
options.inputs = lib.mkOption {
type = lib.types.attrsOf (lib.types.submodule Dependency);
default = { };
};
}
);
Nodes = lib.types.attrsOf (lib.types.submodule Node);
scope-fun = import ./scope-fun.nix {
inherit lib;
inherit (config.dag) nodes;
};
in
{
options.dag = {
nodes = lib.mkOption {
type = Nodes;
description = "Derivation DAG, including cache status and references.";
};
needBuilds = lib.mkOption {
type = with lib.types; nullOr int;
default = null;
description = "How many builds are required to satisfy all targets. Null disables the test";
};
needDownloads = lib.mkOption {
type = with lib.types; nullOr int;
default = null;
description = "How many downloads are required to satisfy all targets. Null disables the test";
};
choseBuilds = lib.mkOption {
type = with lib.types; nullOr int;
default = null;
description = "How many builds we expect evanix to choose to satisfy the maximum number of targets within the given budget. Null disables the test";
};
choseDownloads = lib.mkOption {
type = with lib.types; nullOr int;
default = null;
description = "How many downloads we expect evanix to choose to satisfy the maximum number of targets within the given budget. Null disables the test";
};
allowBuilds = lib.mkOption {
type = with lib.types; nullOr int;
default = null;
description = "How many builds evanix is allowed to choose. Null means no constraint";
};
allowDownloads = lib.mkOption {
type = with lib.types; nullOr int;
default = null;
description = "How many downloads evanix is allowed to choose. Null means no constraint";
};
};
config.nodes.builder =
{ pkgs, ... }:
let
scope = pkgs.lib.makeScope pkgs.newScope scope-fun;
configJson = (pkgs.formats.json { }).generate "nix-dag.json" config.dag;
expressions = pkgs.writeText "guest-scope.nix" ''
let
pkgs = import ${pkgs.path} { };
config = builtins.fromJSON (builtins.readFile ${configJson});
in
pkgs.lib.makeScope pkgs.newScope (pkgs.callPackage ${./scope-fun.nix} { inherit (pkgs) lib; inherit (config) nodes; })
'';
tester = pkgs.writers.writePython3Bin "dag-test" { } ''
# flake8: noqa
import json
import re
import subprocess
import sys
with open("${configJson}", "r") as f:
config = json.load(f)
nodes = config["nodes"]
print(f"config={config}", file=sys.stderr)
def path_to_name(path: str) -> str:
return re.sub(r"^[ ]*${builtins.storeDir}/[a-z0-9]*-([a-zA-Z0-9_-]+)(\.drv)?", r"\1", path)
def parse_dry_run(output):
to_fetch = [ ]
to_build = [ ]
bin = "undefined"
for line in output.split("\n"):
if not line:
continue
if re.match("^.*will be built:$", line):
bin = "to_build"
continue
elif re.match("^.*will be fetched.*:$", line):
bin = "to_fetch"
continue
if not re.match("^[ ]*${builtins.storeDir}", line):
print(f"Skipping line: {line}", file=sys.stderr)
continue
line = path_to_name(line)
if bin == "to_build":
to_build.append(line)
elif bin == "to_fetch":
to_fetch.append(line)
else:
raise RuntimeError("nix-build --dry-run produced invalid output", line)
return to_fetch, to_build
drv_to_schedule = {}
for name, node in nodes.items():
p = subprocess.run(["nix-build", "${expressions}", "--dry-run", "--show-trace", "-A", name], check=True, stderr=subprocess.PIPE)
output = p.stderr.decode("utf-8")
to_fetch, to_build = parse_dry_run(output)
drv_to_schedule[name] = (to_fetch, to_build)
drv_to_action = {}
for (to_fetch, to_build) in drv_to_schedule.values():
for dep in to_fetch:
name = path_to_name(dep)
if name not in drv_to_action:
drv_to_action[name] = "fetch"
assert drv_to_action[name] == "fetch", f"Conflicting plan for {dep}"
for dep in to_build:
name = path_to_name(dep)
if name not in drv_to_action:
drv_to_action[name] = "build"
assert drv_to_action[name] == "build", f"Conflicting plan for {dep}"
print(f"Schedule: {drv_to_action}", file=sys.stderr)
print(f"Per-derivation schedules: {drv_to_schedule}", file=sys.stderr)
for name, node in nodes.items():
error_msg = f"Wrong plan for {name}"
if node["cache"] == "local":
assert name not in drv_to_action, error_msg
elif node["cache"] == "remote":
assert drv_to_action.get(name, None) == "fetch", error_msg
elif node["cache"] == "unbuilt":
assert drv_to_action.get(name, None) == "build", error_msg
need_dls, need_builds = set(), set()
for name, node in nodes.items():
if node["request"]:
need_dls.update(drv_to_schedule[name][0])
need_builds.update(drv_to_schedule[name][1])
if (expected_need_dls := config.get("needDownloads", None)) is not None:
assert len(need_dls) == expected_need_dls, f"{len(need_dls)} != {expected_need_dls}; building {need_dls}"
print("Verified `needDownloads`", file=sys.stderr)
if (expected_need_builds := config.get("needBuilds", None)) is not None:
assert len(need_builds) == expected_need_builds, f"{len(need_builds)} != {expected_need_builds}; building {need_builds}"
print("Verified `needBuilds`", file=sys.stderr)
'';
in
{
system.extraDependencies =
lib.pipe config.dag.nodes [
builtins.attrValues
(builtins.filter ({ cache, ... }: cache == "local"))
(builtins.map ({ name, ... }: scope.${name}))
]
++ [
expressions
pkgs.path
(pkgs.runCommand "any-run-command" { } "").inputDerivation
];
networking.hostName = "builder";
networking.domain = "evanix-tests.local";
nix.settings.substituters = lib.mkForce [ "http://substituter" ];
systemd.tmpfiles.settings."10-expressions" = {
"/run/dag-test/nix-dag-test.json"."L+".argument = "${configJson}";
"/run/dag-test/scope.nix"."L+".argument = "${expressions}";
};
environment.systemPackages = [ tester ];
};
config.nodes.substituter =
{ pkgs, ... }:
let
scope = pkgs.lib.makeScope pkgs.newScope scope-fun;
in
{
system.extraDependencies = lib.pipe config.dag.nodes [
builtins.attrValues
(builtins.filter ({ cache, ... }: cache == "remote"))
(builtins.map ({ name, ... }: scope.${name}))
];
services.nix-serve.enable = true;
services.nix-serve.port = 80;
services.nix-serve.openFirewall = true;
# Allow listening on 80
systemd.services.nix-serve.serviceConfig.User = lib.mkForce "root";
networking.hostName = "substituter";
networking.domain = "evanix-tests.local";
};
}
|
