nextflow提交任务到slurm

学习资料

• nf-core share config

向slurm集群的node4节点申请16核心的CPU，50G的内存提交任务，每次最多提交两个任务

nextflow.config文件的内容如下所示

executor.queueSize = 2
process {
    executor = "slurm"
    clusterOptions = '-p low -w node[4]'
    cpus = 16
    memory = 50.GB
}

scontrol show job 1881485

JobId=1881485 JobName=nf-FASTP_(KY_2311301037310975)
   UserId=wy(1002) GroupId=wy(1002) MCS_label=N/A
   Priority=1 Nice=0 Account=test QOS=normal
   JobState=PENDING Reason=Priority Dependency=(null)
   Requeue=0 Restarts=0 BatchFlag=1 Reboot=0 ExitCode=0:0
   RunTime=00:00:00 TimeLimit=UNLIMITED TimeMin=N/A
   SubmitTime=2023-12-01T17:43:13 EligibleTime=2023-12-01T17:43:13
   AccrueTime=2023-12-01T17:43:13
   StartTime=2023-12-03T17:33:53 EndTime=Unknown Deadline=N/A
   SuspendTime=None SecsPreSuspend=0 LastSchedEval=2023-12-01T17:43:23 Scheduler=Backfill:*
   Partition=high AllocNode:Sid=master:88133
   ReqNodeList=node4 ExcNodeList=(null)
   NodeList=(null)
   NumNodes=1 NumCPUs=16 NumTasks=1 CPUs/Task=16 ReqB:S:C:T=0:0:*:*
   TRES=cpu=16,mem=50G,node=1,billing=16
   Socks/Node=* NtasksPerN:B:S:C=0:0:*:* CoreSpec=*
   MinCPUsNode=16 MinMemoryNode=50G MinTmpDiskNode=0
   Features=(null) DelayBoot=00:00:00
   OverSubscribe=OK Contiguous=0 Licenses=(null) Network=(null)
   Command=.command.run
   WorkDir=/data/wangyang/nextflow/rna-seq/work/57/0edc0c29fd70fd24c9ace0932db593
   StdErr=/data/wangyang/nextflow/rna-seq/work/57/0edc0c29fd70fd24c9ace0932db593/.command.log
   StdIn=/dev/null
   StdOut=/data/wangyang/nextflow/rna-seq/work/57/0edc0c29fd70fd24c9ace0932db593/.command.log
   Power=

使用withLabel属性

executor.queueSize = 2
process {
    executor = "slurm"
    clusterOptions = '-p high -w node[1]'
    cpus = 16
    memory = 50.GB

    withLabel:process_low {
        cpus   = 10 
        memory = 30.GB  
        time   = 4.h  
    }
}

scontrol show job 1886640

JobId=1886640 JobName=nf-FASTP_(KY_2311301037310975)
   UserId=wy(1002) GroupId=wy(1002) MCS_label=N/A
   Priority=1 Nice=0 Account=test QOS=normal
   JobState=RUNNING Reason=None Dependency=(null)
   Requeue=0 Restarts=0 BatchFlag=1 Reboot=0 ExitCode=0:0
   RunTime=00:00:54 TimeLimit=04:00:00 TimeMin=N/A
   SubmitTime=2023-12-04T08:37:41 EligibleTime=2023-12-04T08:37:41
   AccrueTime=2023-12-04T08:37:41
   StartTime=2023-12-04T08:37:42 EndTime=2023-12-04T12:37:42 Deadline=N/A
   PreemptEligibleTime=2023-12-04T08:37:42 PreemptTime=None
   SuspendTime=None SecsPreSuspend=0 LastSchedEval=2023-12-04T08:37:42 Scheduler=Backfill
   Partition=high AllocNode:Sid=master:88133
   ReqNodeList=node1 ExcNodeList=(null)
   NodeList=node1
   BatchHost=node1
   NumNodes=1 NumCPUs=10 NumTasks=1 CPUs/Task=10 ReqB:S:C:T=0:0:*:*
   TRES=cpu=10,mem=30G,node=1,billing=10
   Socks/Node=* NtasksPerN:B:S:C=0:0:*:* CoreSpec=*
   MinCPUsNode=10 MinMemoryNode=30G MinTmpDiskNode=0
   Features=(null) DelayBoot=00:00:00
   OverSubscribe=OK Contiguous=0 Licenses=(null) Network=(null)
   Command=.command.run
   WorkDir=/data/wangyang/nextflow/rna-seq/work/83/4ede7cee0b88b6a5b5b029edbc6315
   StdErr=/data/wangyang/nextflow/rna-seq/work/83/4ede7cee0b88b6a5b5b029edbc6315/.command.log
   StdIn=/dev/null
   StdOut=/data/wangyang/nextflow/rna-seq/work/83/4ede7cee0b88b6a5b5b029edbc6315/.command.log
   Power=

使用Docker 作为driver

调试阶段

docker  run -it wybioinfo/nextflow bash