all.adminpath /tmp/xrootd/var/spool
all.pidpath /tmp/xrootd/var/run
oss.localroot /origin
all.export /
pfc.trace all
ofs.trace all
xrd.trace all -sched
pss.setopt DebugLevel 5
scitokens.trace all
# Enable checksum
xrootd.chksum adler32
# Config TLS
xrd.tls /etc/grid-security/xrd/xrdcert.pem /etc/grid-security/xrd/xrdkey.pem
xrd.tlsca certdir /etc/grid-security/certificates refresh 8h
xrootd.tls capable all -data
sec.level all compatible
all.sitename ucsd
# Enable Security
xrootd.seclib libXrdSec.so
# Enable "gsi" security
#sec.protparm gsi -vomsfun:libXrdVoms.so -vomsfunparms:dbg
#sec.protocol gsi -ca:1 -crl:3 -gridmap:/dev/null
macaroons.secretkey /etc/xrootd/macaroon-secret
# Authorizaton
acc.audit deny
acc.authdb /etc/xrootd/auth_file
acc.authrefresh 60
ofs.authorize 1
# Xrootd TPC using rendezvous key
ofs.tpc logok autorm pgm /etc/xrootd/xrdcp-tpc.sh
# Env var needed by the above TPC script.
setenv X509_USER_CERT = /etc/grid-security/xrd/xrdcert.pem
setenv X509_USER_KEY = /etc/grid-security/xrd/xrdkey.pem
ofs.authlib ++ libXrdAccSciTokens.so config=/etc/xrootd/scitokens.cfg
ofs.authlib ++ libXrdMacaroons.so
xrd.protocol http libXrdHttp.so
http.exthandler xrdtpc libXrdHttpTPC.so
cat /etc/xrootd/macaroon-secret
SPKWus+p1S/dSpk15W9Cu/hCWeM0LnPuiNItzyAhkgUlUkAvzRYSOloI2HCSLKvk
HzWOu3pTjlx1SsG2nyEyCw==
cat /etc/xrootd/auth_file
g /xrootd /data rl
cat /etc/xrootd/scitokens.cfg
[Global]
onmissing = passthrough
[Issuer OSG Monitoring]
issuer = https://osg-htc.org/monitoring
base_path = /
map_subject = false
default_user = xrootd
cat /etc/xrootd/xrdcp-tpc.sh
#!/bin/sh
set -- `getopt S: -S 1 $*`
while [ $# -gt 0 ]
do
case $1 in
-S)
((nstreams=$2-1))
[ $nstreams -ge 1 ] && TCPstreamOpts="-S $nstreams"
shift 2
;;
--)
shift
break
;;
esac
done
src=$1
dst=$2
xrdcp --server $TCPstreamOpts -f $src root://$XRDXROOTD_PROXY/${dst}
Hi Fabio,Thanks for reaching out!This is command I’m using:[centos@xrootd ~]$ curl -v -X PUT --upload-file test.repo -H "Authorization: Bearer $AT" --cacert fullchain.pem https://xrootd.e-commons.chalmers.se:80/data/testfile-token_new.repoIf I comment out “ofs.authorize” - there are no problems in uploading a file:#############################################################################[centos@xrootd ~]$ curl -v -X PUT --upload-file test.repo -H "Authorization: Bearer $AT" --cacert fullchain.pem https://xrootd.e-commons.chalmers.se:80/data/testfile-token_NOauthz.repo* We are completely uploaded and fine* Closing connection 0:-)[centos@xrootd ~ ls -lh /datatotal 16Kdrwxr-xr-x. 2 xrootd xrootd 6 Feb 6 14:13 escape-rw-r--r--. 1 xrootd xrootd 77 Jan 25 13:21 four.txt-rw-r--r--. 1 xrootd xrootd 168 Feb 6 15:22 testfile-token.repo-rw-r--r--. 1 xrootd xrootd 168 Feb 8 13:17 testfile-token_NOauthz.repo-rw-r--r--. 1 xrootd xrootd 168 Feb 2 14:18 testfile.repo[centos@xrootd ~]$##############################################################################I also added more tracing as you suggested.Cheers,Dejan###############################################################################[centos@xrootd ~]$ curl -v -X PUT --upload-file test.repo -H "Authorization: Bearer $AT" --cacert fullchain.pem https://xrootd.e-commons.chalmers.se:80/data/testfile-token_new.repo* About to connect() to xrootd.e-commons.chalmers.se port 80 (#0)* Trying ::1...* Connected to xrootd.e-commons.chalmers.se (::1) port 80 (#0)* Initializing NSS with certpath: sql:/etc/pki/nssdb* CAfile: fullchain.pemCApath: none* NSS: client certificate not found (nickname not specified)* SSL connection using TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384* Server certificate:* subject: CN=xrootd.e-commons.chalmers.se* start date: Jan 30 14:29:24 2023 GMT* expire date: Apr 30 14:29:23 2023 GMT* common name: xrootd.e-commons.chalmers.se* issuer: CN=R3,O=Let's Encrypt,C=US> PUT /data/testfile-token_new.repo HTTP/1.1> User-Agent: curl/7.29.0> Host: xrootd.e-commons.chalmers.se> Accept: */*> Authorization: Bearer eyJraWQiOiJyc2ExIiwiYWxnIjoiUlMyNTYifQ.eyJ3bGNnLnZlciI6IjEuMCIsInN1YiI6IjJkZWJhOWQxLTk1NTgtNDk2My05NWJjLTc1ZTk5M2UzYzgyZSIsImF1ZCI6Imh0dHBzOlwvXC93bGNnLmNlcm4uY2hcL2p3dFwvdjFcL2FueSIsIm5iZiI6MTY3NTg1OTE2Mywic2NvcGUiOiJhZGRyZXNzIG9wZW5pZCBwcm9maWxlIHN0b3JhZ2Uuc3RhZ2U6XC8gZWR1cGVyc29uX2VudGl0bGVtZW50IHBob25lIG9mZmxpbmVfYWNjZXNzIGVkdXBlcnNvbl9zY29wZWRfYWZmaWxpYXRpb24gZWR1cGVyc29uX2Fzc3VyYW5jZSBlbWFpbCB3bGNnLmdyb3VwcyIsImlzcyI6Imh0dHBzOlwvXC9pYW0tZXNjYXBlLmNsb3VkLmNuYWYuaW5mbi5pdFwvIiwiZXhwIjoxNjc1ODYyNzYzLCJpYXQiOjE2NzU4NTkxNjMsImp0aSI6ImIyZTZjOGU0LTZlM2ItNDkzZS04ODcwLWNjNDY0NWYzMTk0OSIsImNsaWVudF9pZCI6ImMyNjU0ZWZiLWY4MTUtNGE2ZS04NDQ4LWIyZWU4MGVjYjE2YiIsIndsY2cuZ3JvdXBzIjpbIlwvZXNjYXBlIiwiXC9lc2NhcGVcL3NrYSJdfQ.L2DKM95W1ovF3QOIPrYR5ifGkyXlDgW2FwiKoSFm2XXAXVdzqrK36gQBCTu2hqoXaP9-6eU_a6Un0jXaY4Gi457HPUk4mDy8Mm0ZctaWAzOZnMyIIbvv0VKmEfFUDq_gBLMr1Lq2PbIuvHbhGhi58dyNlj4pdI8Ped19Q4fNXzg> Content-Length: 168> Expect: 100-continue>< HTTP/1.1 403 Forbidden< Connection: Keep-Alive< Server: XrootD/v5.5.1< Content-Length: 66* HTTP error before end of send, stop sending<Unable to create /data/testfile-token_new.repo; permission denied* Closing connection 0[centos@xrootd ~]$#######################################################################[centos@xrootd ~]$ sudo cat /etc/xrootd/scitokens.cfg[Global]onmissing = passthrough# don't use https://wlcg.cern.ch/jwt/v1/any on production instances# audience = https://xrd.example.com:1094, https://wlcg.cern.ch/jwt/v1/any[Issuer ESCAPE IAM]base_path = /datamap_subject = falsedefault_user = xrootd[centos@xrootd ~]$########################################################################230208 13:32:12 22541 Starting on Linux 3.10.0-1160.81.1.el7.x86_64230208 13:32:12 22541 /usr/bin/xrootd -l /var/log/xrootd/xrootd.log -c /etc/xrootd/xrootd-http.cfg -k fifo -s /run/xrootd/xrootd-http.pid -n httpCopr. 2004-2012 Stanford University, xrd version v5.5.1Config warning: this hostname, localhost, is registered without a domain qualification.++++++ xrootd http@localhost initialization started.Config using configuration file /etc/xrootd/xrootd-http.cfg=====> all.adminpath /var/spool/xrootd=====> all.pidpath /run/xrootd=====> xrd.protocol XrdHttp:80 libXrdHttp.so=====> xrd.tls /etc/grid-security/xrd/xrdcert.pem /etc/grid-security/xrd/xrdkey.pem=====> xrd.tlsca certdir /etc/grid-security/certificates refresh 8h=====> xrd.trace all -sched=====> continue /etc/xrootd/config.d/++++++ xrootd http@localhost TLS initialization started.------ xrootd http@localhost TLS initialization ended.Config maximum number of connections restricted to 65536Config maximum number of threads restricted to 7149230208 13:32:12 22541 Xrd_Config: sendfile enabled.230208 13:32:12 22541 Xrd_LinkCtl: Allocating 64 link objects at a time230208 13:32:12 22541 Xrd_Poll: Starting poller 0230208 13:32:12 22541 Xrd_Poll: Starting poller 1230208 13:32:12 22541 Xrd_Poll: Starting poller 2230208 13:32:12 22541 Xrd_ProtLoad: protocol xroot wants to use port 1094Plugin loaded xrdhttp v5.5.1 from protocol libXrdHttp-5.so230208 13:32:12 22541 Xrd_ProtLoad: protocol XrdHttp wants to use port 80230208 13:32:12 22541 Xrd_Config: xroot:1094 wsz=87380230208 13:32:12 22541 Xrd_ProtLoad: getting protocol object xrootCopr. 2012 Stanford University, xroot protocol 5.1.0 version v5.5.1++++++ xroot protocol initialization started.=====> all.export /data=====> xrootd.tls capable all -data=====> xrootd.seclib libXrdSec.so=====> continue /etc/xrootd/config.d/Config exporting /dataPlugin loaded secprot v5.5.1 from seclib libXrdSec-5.so++++++ Authentication system initialization started.Plugin loaded secztn v5.5.1 from sec.protocol libXrdSecztn-5.so=====> sec.protocol ztn=====> continue /etc/xrootd/config.d/Config 1 authentication directives processed in /etc/xrootd/xrootd-http.cfg------ Authentication system initialization completed.++++++ Protection system initialization started.Config warning: Security level is set to none; request protection disabled!Config Local protection level: noneConfig Remote protection level: none------ Protection system initialization completed.Config Routing for [::1]: local pub4 prv4 pub6 prv6Config Route all4: 127.0.0.1 Dest=[::127.0.0.1]:1094Config Route all6: [::1] Dest=[::1]:1094++++++ File system initialization started.=====> ofs.authlib ++ libXrdAccSciTokens.so config=/etc/xrootd/scitokens.cfg=====> ofs.trace all=====> continue /etc/xrootd/config.d/++++++ Storage system initialization started.=====> all.export /data=====> continue /etc/xrootd/config.d/Config effective /etc/xrootd/xrootd-http.cfg oss configuration:oss.alloc 0 0 0oss.spacescan 600oss.fdlimit 32768 65536oss.maxsize 0oss.trace 0oss.xfr 1 deny 10800 keep 1200oss.memfile off max 963475456oss.defaults r/w nocheck nodread nomig nopurge norcreate nostageoss.path /data r/w nocheck nodread nomig nopurge norcreate nostage------ Storage system initialization completed.++++++ Authorization system initialization started.230208 13:32:12 22541 acc_Config: Authorization system using configuration in /etc/xrootd/xrootd-http.cfg=====> acc.authdb /etc/xrootd/Authfile=====> continue /etc/xrootd/config.d/Config 1 authorization directives processed in /etc/xrootd/xrootd-http.cfgConfig 1 auth entries processed in /etc/xrootd/Authfile------ Authorization system initialization completed.Plugin loaded XrdAccSciTokens v5.5.1 from authlib libXrdAccSciTokens-5.so++++++ XrdAccSciTokens: Initialized SciTokens-based authorization.=====> scitokens.trace all=====> continue /etc/xrootd/config.d/230208 13:32:12 22541 scitokens_Config: Logging levels enabled - all230208 13:32:12 22541 scitokens_Reconfig: Parsing configuration file: /etc/xrootd/scitokens.cfg230208 13:32:12 22541 scitokens_Reconfig: Configuring issuer https://iam-escape.cloud.cnaf.infn.it/++++++ Checkpoint initialization started.++++++ Checkpoint initialization completed.Config effective /etc/xrootd/xrootd-http.cfg ofs configuration:all.role serverofs.authorizeofs.maxdelay 60ofs.persist manual hold 600 logdir /var/spool/xrootd/http/.ofs/posc.logofs.trace ffffofs.authlib defaultofs.authlib ++ libXrdAccSciTokens.so config=/etc/xrootd/scitokens.cfg------ File system server initialization completed.Config asynchronous I/O has been disabled!230208 13:32:12 22541 ofs_FAttr: FAttr req=info------ xroot protocol initialization completed.230208 13:32:12 22541 Xrd_ProtLoad: enabling port 1094 for protocol xroot230208 13:32:12 22541 Xrd_Config: XrdHttp:80 wsz=87380230208 13:32:12 22541 Xrd_ProtLoad: getting protocol object XrdHttpCopr. 2012 CERN IT, an HTTP implementation for the XRootD framework.++++++ HTTP protocol initialization started.=====> http.header2cgi Authorization authz=====> continue /etc/xrootd/config.d/Config Using xrd.tls to supply 'cert' and 'key'.Config Using xrd.tlsca to supply 'cadir'.++++++ HTTPS initialization started.------ HTTPS initialization completed.230208 13:32:12 22541 sysConfig: XRDROLE: server230208 13:32:12 22541 sysConfig: Configured as HTTP(s) data server.------ HTTP protocol initialization completed.230208 13:32:12 22541 Xrd_ProtLoad: enabling port 80 for protocol XrdHttp------ xrootd http@localhost:80 initialization completed.230208 13:32:12 22558 TLS_Refresh: CRL refresh started.230208 13:32:12 22558 TLS_Refresh: CRL refresh will happen in 28800 seconds.230208 13:32:12 22550 TLS_Refresh: CRL refresh started.230208 13:32:12 22550 TLS_Refresh: CRL refresh will happen in 28800 seconds.230208 13:32:32 22546 Xrd_Inet: Accepted connection on port 80 from 25@localhost230208 13:32:32 22546 Xrd_ProtLoad: matched port 80 protocol XrdHttp230208 13:32:32 22546 anon.0:25@localhost Xrd_Poll: FD 25 attached to poller 0; num=1230208 13:32:32 22546 XrootdBridge: unknown.1:25@localhost login as nobody230208 13:32:32 22546 unknown.1:25@localhost ofs_open: 200-40664 fn=/data/testfile-token_WITHauthz.repo230208 13:32:32 22546 scitokens_Access: Trying token-based access control230208 13:32:32 22546 scitokens_Access: Token not found in recent cache; parsing.230208 13:32:32 22546 scitokens_Access: New valid token mapped_username=xrootd, subject=2deba9d1-9558-4963-95bc-75e993e3c82e, issuer=https://iam-escape.cloud.cnaf.infn.it/, groups=/escape,/escape/ska230208 13:32:32 22546 scitokens_Access: Trying token-based access control230208 13:32:32 22546 scitokens_Access: Cached token mapped_username=xrootd, subject=2deba9d1-9558-4963-95bc-75e993e3c82e, issuer=https://iam-escape.cloud.cnaf.infn.it/, groups=/escape,/escape/ska230208 13:32:32 22546 ofs_open: unknown.1:25@localhost Unable to create /data/testfile-token_WITHauthz.repo; permission denied230208 13:32:32 22546 unknown.1:25@localhost ofs_close: use=0 fn=dummy230208 13:32:32 22546 XrootdXeq: unknown.1:25@localhost disc 0:00:00 (send failure)230208 13:32:32 22546 unknown.1:25@localhost Xrd_Poll: Poller 0 removing FD 25230208 13:32:32 22546 unknown.1:25@localhost Xrd_Poll: FD 25 detached from poller 0; num=0[centos@xrootd ~]$On 8 Feb 2023, at 12:42, Fabio Andrijauskas <[log in to unmask]> wrote:Hi,Can you send how are you requesting/creating the files?Try to add this to get all the possible logs:pfc.trace allofs.trace allxrd.trace all -schedpss.setopt DebugLevel 5scitokens.trace allDid you try to create the the path on the export path?Can you send your sci token config file?
--Hi Matt,
I run out of ideas, so all suggestions are appreciated.
I think unix bits are in place.
Cheers,
Dejan
……
[centos@xrootd ~]$ ls -lh / |grep data
drwxr-xr-x. 3 xrootd xrootd 84 Feb 6 15:22 data
[centos@xrootd ~]$ ls -lh /data
total 12K
drwxr-xr-x. 2 xrootd xrootd 6 Feb 6 14:13 escape
-rw-r--r--. 1 xrootd xrootd 77 Jan 25 13:21 four.txt
-rw-r--r--. 1 xrootd xrootd 168 Feb 6 15:22 testfile-token.repo
-rw-r--r--. 1 xrootd xrootd 168 Feb 2 14:18 testfile.repo
[centos@xrootd ~]$
> On 8 Feb 2023, at 11:05, Doidge, Matt <[log in to unmask]> wrote:
>
> Hello,
> This is a bit of a low-level suggestion, but can the xrootd unix user write to /data? I had similar looking issues with a test xrootd server and a known working config where I had forgotten to chown the exported path.
>
> My apologies for the noise if you've already checked this.
>
> Cheers,
> Matt
>
> ________________________________________
> From: [log in to unmask] <[log in to unmask]> on behalf of Dejan Vitlacil <[log in to unmask]>
> Sent: 08 February 2023 09:24
> To: Oliver Freyermuth
> Cc: [log in to unmask]
> Subject: [External] Re: XRootD and tokens
>
> This email originated outside the University. Check before clicking links or attachments.
########################################################################
Use REPLY-ALL to reply to list
To unsubscribe from the XROOTD-L list, click the following link:
https://listserv.slac.stanford.edu/cgi-bin/wa?SUBED1=XROOTD-L&A=1
--
Fábio Andrijauskas
Use REPLY-ALL to reply to list
To unsubscribe from the XROOTD-L list, click the following link:
https://listserv.slac.stanford.edu/cgi-bin/wa?SUBED1=XROOTD-L&A=1