Skip to content

Commit 1d3f3db

Browse files
hariharan-devarajanhariharan-devarajan
authored
Merge pull request #88 from LLNL/feature/ci
Feature: Added CI for making sure changes are tested.
2 parents d58d0c1 + 466ae75 commit 1d3f3db

10 files changed

Lines changed: 439 additions & 93 deletions

File tree

‎.github/workflows/docker-build-test.yml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: Docker Build and Test
2+
3+
on:
4+
push:
5+
pull_request:
6+
branches: [ develop, stage-in ]
7+
workflow_dispatch:
8+
9+
env:
10+
DOCKER_IMAGE_NAME: datacrumbs-test
11+
DOCKER_TAG: latest
12+
13+
jobs:
14+
docker-build-and-test:
15+
runs-on: ubuntu-latest
16+
17+
steps:
18+
- name: Checkout repository
19+
uses: actions/checkout@v4
20+
with:
21+
submodules: recursive
22+
23+
- name: Set up Docker Buildx
24+
uses: docker/setup-buildx-action@v3
25+
26+
- name: Build Docker image
27+
run: |
28+
docker buildx build \
29+
--cache-from=type=gha \
30+
--cache-to=type=gha,mode=max \
31+
--load \
32+
-t ${{ env.DOCKER_IMAGE_NAME }}:${{ env.DOCKER_TAG }} \
33+
-f infrastrutcure/docker/Dockerfile \
34+
.
35+
36+
- name: Run DataCrumbs tests
37+
run: |
38+
docker run --rm \
39+
-v /lib/modules/:/lib/modules:ro \
40+
-v /sys/kernel/debug/:/sys/kernel/debug:rw \
41+
--net=host \
42+
--pid=host \
43+
--privileged \
44+
--cap-add sys_admin \
45+
--cap-add sys_ptrace \
46+
--hostname docker \
47+
-v /sys/fs/bpf:/sys/fs/bpf \
48+
${{ env.DOCKER_IMAGE_NAME }}:${{ env.DOCKER_TAG }} \
49+
bash -c /opt/datacrumbs/infrastrutcure/docker/test_run.sh

‎CMakeLists.txt‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,17 @@ endif()
307307
if(LIBBPF_FOUND)
308308
include_directories(${LIBBPF_INCLUDEDIR})
309309
link_directories(${LIBBPF_LIBRARY_DIRS})
310+
311+
# If LIBBPF_LIBRARY_DIRS is not set, try LIBBPF_LIBDIR, else get parent dir of LIBBPF_LINK_LIBRARIES
312+
if(NOT LIBBPF_LIBRARY_DIRS)
313+
if(LIBBPF_LIBDIR)
314+
set(LIBBPF_LIBRARY_DIRS "${LIBBPF_LIBDIR}")
315+
elseif(LIBBPF_LINK_LIBRARIES)
316+
get_filename_component(_LIBBPF_LIB_PARENT "${LIBBPF_LINK_LIBRARIES}" DIRECTORY)
317+
set(LIBBPF_LIBRARY_DIRS "${_LIBBPF_LIB_PARENT}")
318+
endif()
319+
endif()
320+
310321
list(APPEND DEPENDENCY_LIBRARY_DIRS ${LIBBPF_LIBRARY_DIRS})
311322
set(DEPENDENCY_LIB ${DEPENDENCY_LIB} -L${LIBBPF_LIBRARY_DIRS} -lbpf -lelf)
312323
else()
@@ -323,7 +334,7 @@ if(${yaml-cpp_FOUND})
323334
endif()
324335

325336
list(APPEND DEPENDENCY_LIBRARY_DIRS ${YAML_CPP_LIBRARY_DIR})
326-
set(DEPENDENCY_LIB ${DEPENDENCY_LIB} -L${YAML_CPP_LIBRARY_DIR} ${YAML_CPP_LIBRARIES})
337+
set(DEPENDENCY_LIB ${DEPENDENCY_LIB} -L${YAML_CPP_LIBRARY_DIR} yaml-cpp)
327338
else()
328339
message(FATAL_ERROR "-- [${UPPER_PROJECT_NAME}] yaml-cpp is needed for ${PROJECT_NAME} build")
329340
endif()

‎etc/datacrumbs/configs/docker-toss.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
name: lead2
2+
install_dir: /workspaces/datacrumbs/install
3+
mode: tracer
4+
trace_file_path: /scratch/haridev/logs/lead2-trace.pfw
5+
capture_probes:
6+
- name: custom1
7+
start_event_id: 100000
8+
probe: custom
9+
type: custom
10+
file: /opt/datacrumbs/etc/datacrumbs/plugins/custom_probes/sys_io/sysio.bpf.c
11+
probes: /opt/datacrumbs/etc/datacrumbs/plugins/custom_probes/sys_io/probes.json
12+
process_header: /opt/datacrumbs/etc/datacrumbs/plugins/custom_probes/sys_io/sysio_process.h
13+
event_type: 2
14+
- name: libc
15+
probe: uprobe
16+
type: binary
17+
file: /usr/lib64/libc.so.6
18+
regex: .*fork.*
19+
- name: python
20+
probe: usdt
21+
type: usdt
22+
binary_path: /lib64/libpython3.6m.so.1.0
23+
provider: python
24+
- name: os_page_cache
25+
probe: kprobe
26+
type: header
27+
file: /usr/src/kernels/4.18.0-553.8.1.1toss.t4.x86_64/include/linux/pagemap.h
28+
- name: shm
29+
probe: kprobe
30+
type: header
31+
file: /usr/src/kernels/4.18.0-553.8.1.1toss.t4.x86_64/include/linux/shmem_fs.h
32+
- name: iomap
33+
probe: kprobe
34+
type: ksym
35+
regex: ^iomap(?!.*cold).*
36+
profiling_interval: 0.0001

‎etc/datacrumbs/configs/docker.yaml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@ capture_probes:
77
start_event_id: 100000
88
probe: custom
99
type: custom
10-
file: /workspaces/datacrumbs/etc/datacrumbs/plugins/custom_probes/sys_io/sysio.bpf.c
11-
probes: /workspaces/datacrumbs/etc/datacrumbs/plugins/custom_probes/sys_io/probes.json
12-
process_header: /workspaces/datacrumbs/etc/datacrumbs/plugins/custom_probes/sys_io/sysio_process.h
10+
file: /opt/datacrumbs/etc/datacrumbs/plugins/custom_probes/sys_io/sysio.bpf.c
11+
probes: /opt/datacrumbs/etc/datacrumbs/plugins/custom_probes/sys_io/probes.json
12+
process_header: /opt/datacrumbs/etc/datacrumbs/plugins/custom_probes/sys_io/sysio_process.h
1313
event_type: 2
1414
- name: libc
1515
probe: uprobe
@@ -21,16 +21,16 @@ capture_probes:
2121
type: usdt
2222
binary_path: /lib64/libpython3.6m.so.1.0
2323
provider: python
24+
- name: iomap
25+
probe: kprobe
26+
type: ksym
27+
regex: ^iomap(?!.*cold).*
2428
- name: os_page_cache
2529
probe: kprobe
2630
type: header
27-
file: /usr/src/kernels/4.18.0-348.7.1.el8_5.aarch64/include/linux/pagemap.h
31+
file: /usr/src/kernels/4.18.0-348.7.1.el8_5.x86_64/include/linux/pagemap.h
2832
- name: shm
2933
probe: kprobe
3034
type: header
31-
file: /usr/src/kernels/4.18.0-348.7.1.el8_5.aarch64/include/linux/shmem_fs.h
32-
- name: iomap
33-
probe: kprobe
34-
type: ksym
35-
regex: ^iomap(?!.*cold).*
35+
file: /usr/src/kernels/4.18.0-348.7.1.el8_5.x86_64/include/linux/shmem_fs.h
3636
profiling_interval: 0.0001

‎infrastrutcure/docker/Dockerfile‎

Lines changed: 12 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,70 +1,18 @@
1-
FROM centos:centos8
1+
FROM hdevarajan92/datacrumbs:latest
22

3-
RUN cd /etc/yum.repos.d/
4-
RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
5-
RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
3+
RUN mkdir -p /opt/datacrumbs
4+
COPY . /opt/datacrumbs/
5+
RUN . /opt/rh/gcc-toolset-11/enable && \
6+
mkdir -p datacrumbs-build && \
7+
rm -rf datacrumbs-build/* && \
8+
cd datacrumbs-build && \
9+
cmake -DDATACRUMBS_HOST=docker -DDATACRUMBS_USER=root /opt/datacrumbs/ && \
10+
make -j
611

12+
ENV PATH=/opt/datacrumbs/datacrumbs-build/bin:${PATH}
13+
ENV LD_LIBRARY_PATH="/opt/datacrumbs/datacrumbs-build/lib:${LD_LIBRARY_PATH}":${LD_LIBRARY_PATH}
714

8-
RUN yum update -y
9-
10-
RUN yum install -y \
11-
gcc \
12-
clang \
13-
llvm \
14-
elfutils-libelf-devel \
15-
kernel-devel \
16-
kernel-headers \
17-
make \
18-
iproute \
19-
iputils \
20-
git \
21-
vim-enhanced \
22-
which \
23-
python3-pip \
24-
cmake \
25-
llvm-devel.aarch64 \
26-
mpich-devel.aarch64
27-
28-
RUN dnf update -y
29-
30-
RUN dnf group install -y "Development Tools"
31-
RUN dnf install -y clang-devel.aarch64 \
32-
gcc-toolset-11 \
33-
jq
34-
RUN scl enable gcc-toolset-11 bash
35-
RUN git clone --recurse-submodules https://github.com/libbpf/bpftool.git /workspaces/bpftool
36-
RUN cd /workspaces/bpftool && \
37-
git checkout tags/v7.5.0 && \
38-
cd libbpf && \
39-
git checkout tags/v1.5.0 && \
40-
cd /workspaces/bpftool/libbpf/src && \
41-
make && \
42-
make install && \
43-
cd /workspaces/bpftool/src && \
44-
make && \
45-
make install
46-
47-
RUN git clone https://github.com/jbeder/yaml-cpp.git /workspaces/yaml-cpp
48-
RUN cd /workspaces/yaml-cpp && \
49-
git checkout tags/yaml-cpp-0.7.0 && \
50-
mkdir build && \
51-
cd build && \
52-
cmake -DYAML_BUILD_SHARED_LIBS=ON -DYAML_CPP_BUILD_TESTS=OFF .. && \
53-
make -j && \
54-
make install
55-
56-
RUN git clone https://github.com/json-c/json-c.git /workspaces/json-c
57-
RUN cd /workspaces/json-c && \
58-
git checkout tags/json-c-0.15-20200726 && \
59-
mkdir build && \
60-
cd build && \
61-
cmake .. && \
62-
make -j && \
63-
make install
64-
ENV CC=/usr/lib64/mpich/bin/mpicc
65-
ENV CXX=/usr/lib64/mpich/bin/mpic++
66-
# Set working directory
67-
WORKDIR /workspace
15+
WORKDIR /opt/datacrumbs
6816

6917
# By default, run bash. eBPF programs require --privileged and --cap-add=ALL when running the container.
7018
CMD ["/bin/bash"]

‎infrastrutcure/docker/Dockerfile.build‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
FROM centos:centos8
2+
3+
# Update CentOS 8 repositories (CentOS 8 is EOL, use vault)
4+
RUN cd /etc/yum.repos.d/ && \
5+
sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \
6+
sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
7+
8+
# Update system
9+
RUN yum update -y
10+
11+
# Install base development tools and dependencies
12+
RUN yum install -y \
13+
gcc \
14+
clang \
15+
llvm \
16+
elfutils-libelf-devel \
17+
kernel-devel \
18+
kernel-headers \
19+
make \
20+
iproute \
21+
iputils \
22+
git \
23+
vim-enhanced \
24+
which \
25+
python3-pip \
26+
cmake \
27+
llvm-devel \
28+
clang-devel \
29+
gcc-toolset-11 \
30+
jq \
31+
time
32+
33+
# Install Development Tools group
34+
RUN dnf group install -y "Development Tools"
35+
36+
# Set up working directory
37+
WORKDIR /opt
38+
39+
# Build and install bpftool and libbpf
40+
RUN . /opt/rh/gcc-toolset-11/enable && \
41+
git clone --recurse-submodules https://github.com/libbpf/bpftool.git && \
42+
cd bpftool && \
43+
git checkout tags/v7.5.0 && \
44+
cd libbpf && \
45+
git checkout tags/v1.5.0 && \
46+
cd /opt/bpftool/libbpf/src && \
47+
make -j && \
48+
make install && \
49+
cd /opt/bpftool/src && \
50+
make -j && \
51+
make install
52+
53+
# Build and install yaml-cpp
54+
RUN . /opt/rh/gcc-toolset-11/enable && \
55+
git clone https://github.com/jbeder/yaml-cpp.git && \
56+
cd yaml-cpp && \
57+
git checkout tags/yaml-cpp-0.7.0 && \
58+
mkdir build && \
59+
cd build && \
60+
cmake -DYAML_BUILD_SHARED_LIBS=ON -DYAML_CPP_BUILD_TESTS=OFF .. && \
61+
make -j && \
62+
make install
63+
64+
# Build and install json-c
65+
RUN . /opt/rh/gcc-toolset-11/enable && \
66+
git clone https://github.com/json-c/json-c.git && \
67+
cd json-c && \
68+
git checkout tags/json-c-0.15-20200726 && \
69+
mkdir build && \
70+
cd build && \
71+
cmake .. && \
72+
make -j && \
73+
make install
74+
75+
# Update library path
76+
RUN echo "/usr/local/lib64" > /etc/ld.so.conf.d/local.conf && \
77+
echo "/usr/local/lib" >> /etc/ld.so.conf.d/local.conf && \
78+
ldconfig
79+
80+
# Create datacrumbs directory
81+
WORKDIR /opt/datacrumbs
82+
83+
# Set environment variables
84+
ENV PATH="/opt/rh/gcc-toolset-11/root/usr/bin:${PATH}"
85+
ENV LD_LIBRARY_PATH="/opt/rh/gcc-toolset-11/root/usr/lib64:/usr/local/lib64:/usr/local/lib:${LD_LIBRARY_PATH}"
86+
87+
# Create entrypoint script
88+
RUN echo '#!/bin/bash' > /entrypoint.sh && \
89+
echo '. /opt/rh/gcc-toolset-11/enable' >> /entrypoint.sh && \
90+
echo 'exec "$@"' >> /entrypoint.sh && \
91+
chmod +x /entrypoint.sh
92+
93+
# Set the entrypoint
94+
ENTRYPOINT ["/entrypoint.sh"]
95+
96+
# By default, run bash
97+
CMD ["/bin/bash"]
98+
99+
# Add labels for better container management
100+
LABEL maintainer="DataCrumbs Team"
101+
LABEL description="DataCrumbs eBPF-based data provenance tracking system"
102+
LABEL version="1.0"

‎infrastrutcure/docker/README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Notes
2+
3+
```bash
4+
5+
```

‎infrastrutcure/docker/test_run.sh‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
/opt/datacrumbs-build/bin/datacrumbs docker --user root \
3+
--config_path /opt/datacrumbs/etc/datacrumbs/configs \
4+
--data_dir /opt/datacrumbs/etc/datacrumbs/data \
5+
--trace_log_dir /opt/datacrumbs/etc/datacrumbs/logs &
6+
DATACRUMBS_PID=$!
7+
sleep 60
8+
mkdir -p /opt/data/
9+
rm -rf /opt/data/*
10+
LD_PRELOAD=/opt/datacrumbs-build/lib64/libdatacrumbs_client.so dd if=/dev/zero of=/opt/data/img_temp.bat bs=1M count=16
11+
kill -SIGINT $DATACRUMBS_PID
12+
wait $DATACRUMBS_PID
13+
zcat /opt/datacrumbs/etc/datacrumbs/logs/*
14+
exit 0

0 commit comments

Comments
 (0)