{"id":55,"date":"2022-10-10T15:37:34","date_gmt":"2022-10-10T13:37:34","guid":{"rendered":"https:\/\/www.syntera.ch\/blog\/?p=55"},"modified":"2024-07-18T15:38:57","modified_gmt":"2024-07-18T13:38:57","slug":"copy-files-from-sharepoint-to-blob-storage-using-azure-data-factory","status":"publish","type":"post","link":"https:\/\/www.syntera.ch\/blog\/2022\/10\/10\/copy-files-from-sharepoint-to-blob-storage-using-azure-data-factory\/","title":{"rendered":"Copy files from SharePoint to Blob Storage using Azure Data Factory"},"content":{"rendered":"\n<div class=\"wp-block-columns is-layout-flex wp-container-core-columns-is-layout-f56f613f wp-block-columns-is-layout-flex\">\n<div class=\"wp-block-column is-layout-flow wp-block-column-is-layout-flow\"><div class=\"wp-block-post-author has-medium-font-size\"><div class=\"wp-block-post-author__avatar\"><img alt='' src='https:\/\/secure.gravatar.com\/avatar\/9cbefca9184e6b8146d83f72900473b791b31bc6c2a10a8e57f77e9c41f11b45?s=48&#038;d=mm&#038;r=g' srcset='https:\/\/secure.gravatar.com\/avatar\/9cbefca9184e6b8146d83f72900473b791b31bc6c2a10a8e57f77e9c41f11b45?s=96&#038;d=mm&#038;r=g 2x' class='avatar avatar-48 photo' height='48' width='48' \/><\/div><div class=\"wp-block-post-author__content\"><p class=\"wp-block-post-author__byline\">LEAD BUSINESS INTELLIGENCE<\/p><p class=\"wp-block-post-author__name\">Dimitri B\u00fctikofer<\/p><\/div><\/div><\/div>\n\n\n\n<div class=\"wp-block-column is-vertically-aligned-top is-layout-flow wp-block-column-is-layout-flow\">\n<ul class=\"wp-block-outermost-social-sharing alignright has-small-icon-size has-icon-color is-style-logos-only is-content-justification-left is-layout-flex wp-container-outermost-social-sharing-is-layout-fa5e4718 wp-block-outermost-social-sharing-is-layout-flex\"><li style=\"color: #1a4548\" class=\"outermost-social-sharing-link outermost-social-sharing-link-linkedin has-primary-color wp-block-outermost-social-sharing-link\">\n\t<a href=\"https:\/\/www.linkedin.com\/shareArticle?mini=true&#038;url=https%3A%2F%2Fwww.syntera.ch%2Fblog%2F2022%2F10%2F10%2Fcopy-files-from-sharepoint-to-blob-storage-using-azure-data-factory%2F&#038;title=Copy%20files%20from%20SharePoint%20to%20Blob%20Storage%20using%20Azure%20Data%20Factory\" aria-label=\"Share on LinkedIn\" rel=\"noopener nofollow\" target=\"_blank\" class=\"wp-block-outermost-social-sharing-link-anchor\">\n\t\t<svg width=\"24\" height=\"24\" viewBox=\"0 0 24 24\" version=\"1.1\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" aria-hidden=\"true\" focusable=\"false\"><path d=\"M19.7,3H4.3C3.582,3,3,3.582,3,4.3v15.4C3,20.418,3.582,21,4.3,21h15.4c0.718,0,1.3-0.582,1.3-1.3V4.3 C21,3.582,20.418,3,19.7,3z M8.339,18.338H5.667v-8.59h2.672V18.338z M7.004,8.574c-0.857,0-1.549-0.694-1.549-1.548 c0-0.855,0.691-1.548,1.549-1.548c0.854,0,1.547,0.694,1.547,1.548C8.551,7.881,7.858,8.574,7.004,8.574z M18.339,18.338h-2.669 v-4.177c0-0.996-0.017-2.278-1.387-2.278c-1.389,0-1.601,1.086-1.601,2.206v4.249h-2.667v-8.59h2.559v1.174h0.037 c0.356-0.675,1.227-1.387,2.526-1.387c2.703,0,3.203,1.779,3.203,4.092V18.338z\"><\/path><\/svg>\t\t<span class=\"wp-block-outermost-social-sharing-link-label screen-reader-text\">\n\t\t\tShare on LinkedIn\t\t<\/span>\n\t<\/a>\n<\/li>\n\n\n<li style=\"color: #1a4548\" class=\"outermost-social-sharing-link outermost-social-sharing-link-mail has-primary-color wp-block-outermost-social-sharing-link\">\n\t<a href=\"mailto:?subject=Copy%20files%20from%20SharePoint%20to%20Blob%20Storage%20using%20Azure%20Data%20Factory&#038;body=Copy%20files%20from%20SharePoint%20to%20Blob%20Storage%20using%20Azure%20Data%20Factory%20&mdash;%20https%3A%2F%2Fwww.syntera.ch%2Fblog%2F2022%2F10%2F10%2Fcopy-files-from-sharepoint-to-blob-storage-using-azure-data-factory%2F\" aria-label=\"Email this Page\"  class=\"wp-block-outermost-social-sharing-link-anchor\">\n\t\t<svg width=\"24\" height=\"24\" viewBox=\"0 0 24 24\" version=\"1.1\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" aria-hidden=\"true\" focusable=\"false\"><path d=\"M20,4H4C2.895,4,2,4.895,2,6v12c0,1.105,0.895,2,2,2h16c1.105,0,2-0.895,2-2V6C22,4.895,21.105,4,20,4z M20,8.236l-8,4.882 L4,8.236V6h16V8.236z\"><\/path><\/svg>\t\t<span class=\"wp-block-outermost-social-sharing-link-label screen-reader-text\">\n\t\t\tEmail this Page\t\t<\/span>\n\t<\/a>\n<\/li>\n\n\n<li style=\"color: #1a4548\" class=\"outermost-social-sharing-link outermost-social-sharing-link-whatsapp has-primary-color wp-block-outermost-social-sharing-link\">\n\t<a href=\"https:\/\/api.whatsapp.com\/send?text=Copy%20files%20from%20SharePoint%20to%20Blob%20Storage%20using%20Azure%20Data%20Factory%20&mdash;%20https%3A%2F%2Fwww.syntera.ch%2Fblog%2F2022%2F10%2F10%2Fcopy-files-from-sharepoint-to-blob-storage-using-azure-data-factory%2F\" aria-label=\"Share on WhatsApp\" rel=\"noopener nofollow\" target=\"_blank\" class=\"wp-block-outermost-social-sharing-link-anchor\">\n\t\t<svg width=\"24\" height=\"24\" viewBox=\"0 0 24 24\" version=\"1.1\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" aria-hidden=\"true\" focusable=\"false\"><path d=\"M 12.011719 2 C 6.5057187 2 2.0234844 6.478375 2.0214844 11.984375 C 2.0204844 13.744375 2.4814687 15.462563 3.3554688 16.976562 L 2 22 L 7.2324219 20.763672 C 8.6914219 21.559672 10.333859 21.977516 12.005859 21.978516 L 12.009766 21.978516 C 17.514766 21.978516 21.995047 17.499141 21.998047 11.994141 C 22.000047 9.3251406 20.962172 6.8157344 19.076172 4.9277344 C 17.190172 3.0407344 14.683719 2.001 12.011719 2 z M 12.009766 4 C 14.145766 4.001 16.153109 4.8337969 17.662109 6.3417969 C 19.171109 7.8517969 20.000047 9.8581875 19.998047 11.992188 C 19.996047 16.396187 16.413812 19.978516 12.007812 19.978516 C 10.674812 19.977516 9.3544062 19.642812 8.1914062 19.007812 L 7.5175781 18.640625 L 6.7734375 18.816406 L 4.8046875 19.28125 L 5.2851562 17.496094 L 5.5019531 16.695312 L 5.0878906 15.976562 C 4.3898906 14.768562 4.0204844 13.387375 4.0214844 11.984375 C 4.0234844 7.582375 7.6067656 4 12.009766 4 z M 8.4765625 7.375 C 8.3095625 7.375 8.0395469 7.4375 7.8105469 7.6875 C 7.5815469 7.9365 6.9355469 8.5395781 6.9355469 9.7675781 C 6.9355469 10.995578 7.8300781 12.182609 7.9550781 12.349609 C 8.0790781 12.515609 9.68175 15.115234 12.21875 16.115234 C 14.32675 16.946234 14.754891 16.782234 15.212891 16.740234 C 15.670891 16.699234 16.690438 16.137687 16.898438 15.554688 C 17.106437 14.971687 17.106922 14.470187 17.044922 14.367188 C 16.982922 14.263188 16.816406 14.201172 16.566406 14.076172 C 16.317406 13.951172 15.090328 13.348625 14.861328 13.265625 C 14.632328 13.182625 14.464828 13.140625 14.298828 13.390625 C 14.132828 13.640625 13.655766 14.201187 13.509766 14.367188 C 13.363766 14.534188 13.21875 14.556641 12.96875 14.431641 C 12.71875 14.305641 11.914938 14.041406 10.960938 13.191406 C 10.218937 12.530406 9.7182656 11.714844 9.5722656 11.464844 C 9.4272656 11.215844 9.5585938 11.079078 9.6835938 10.955078 C 9.7955938 10.843078 9.9316406 10.663578 10.056641 10.517578 C 10.180641 10.371578 10.223641 10.267562 10.306641 10.101562 C 10.389641 9.9355625 10.347156 9.7890625 10.285156 9.6640625 C 10.223156 9.5390625 9.737625 8.3065 9.515625 7.8125 C 9.328625 7.3975 9.131125 7.3878594 8.953125 7.3808594 C 8.808125 7.3748594 8.6425625 7.375 8.4765625 7.375 z\"><\/path><\/svg>\t\t<span class=\"wp-block-outermost-social-sharing-link-label screen-reader-text\">\n\t\t\tShare on WhatsApp\t\t<\/span>\n\t<\/a>\n<\/li>\n<\/ul>\n<\/div>\n<\/div>\n\n\n\n<p class=\"wp-block-paragraph\">Azure Data Factory is a great tool for ETL pipelines, and we love working with it. However, when it comes to the integration with the rest of the non-Azure Microsoft world (especially SharePoint) it can get a bit frustrating. In a recent project I wanted to build a solution, which allows employees to upload documents from their phones via OneDrive app and its integrated scan function. The uploaded documents should then be fetched and copied to Azure Data Lake Storage using an Azure Data Factory pipeline. Seemed like a simple task until I ran into several not so simple problems. In this blog post I want to share the final solution and walk through all the necessary steps.<\/p>\n\n\n\n<p class=\"wp-block-paragraph\">Addendum from 15.12.2022: In the meantime I expanded the solution, enabling it to copy multiple files from multiple SharePoint folders to an Azure Blob Storage, using a nested loop. If you are interested in a step-by-step guide on this topic you can find the blog here: <a href=\"https:\/\/www.syntera.ch\/blog\/2022\/12\/15\/nested-foreach-loops-in-azure-data-factory\/\">Nested ForEach loops in Azure Data Factory &#8211; Syntera<\/a>. If you only need to copy multiple files of one SharePoint folder the solution shown in this blog will be sufficient.<\/p>\n\n\n\n<h2 class=\"wp-block-heading\">The Solution<\/h2>\n\n\n\n<p class=\"wp-block-paragraph\">Let&#8217;s first have a look at the final pipeline in Azure Data Factory and break it down into individual tasks to solve:<\/p>\n\n\n\n<figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"985\" height=\"330\" src=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/14_architecture.png\" alt=\"\" class=\"wp-image-82\" srcset=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/14_architecture.png 985w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/14_architecture-300x101.png 300w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/14_architecture-768x257.png 768w\" sizes=\"auto, (max-width: 985px) 100vw, 985px\" \/><\/figure>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Prerequisite:<\/strong><\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>Register SharePoint (SPO) application in Azure Active Directory (AAD).<\/li>\n\n\n\n<li>Grant SPO site permission to registered application in AAD.<\/li>\n\n\n\n<li>Provision ResourceGroup, Azure Data Factory (ADF) and Azure Data Lake Storage (ADLS) in your Azure Subscription.<\/li>\n<\/ol>\n\n\n\n<p class=\"wp-block-paragraph\"><strong>Inside of ADF pipeline:<\/strong><\/p>\n\n\n\n<ol start=\"4\" class=\"wp-block-list\">\n<li>WebActivity &#8220;GetBearerToken&#8221;: Get an access token from SPO via API call.<\/li>\n\n\n\n<li>WebActivity &#8220;GetSPOFolderMetadata&#8221;: Get SPO folder metadata including a list of all files in the SPO target folder using the SPO access token via API call.<\/li>\n\n\n\n<li>CopyActivity &#8220;Copy data from SPO to ADLS&#8221; inside ForEach loop: Iterate through files in list of files and copy each file to ADLS.<\/li>\n<\/ol>\n\n\n\n<h4 class=\"wp-block-heading\">1. Register SPO application in AAD<\/h4>\n\n\n\n<p class=\"wp-block-paragraph\">To enable ADF to access content from SPO we need to register an application in AAD. Basically, we are creating an identity inside of our Azure environment and in a second step telling SPO that if someone with this identity ever shows up, he\/she can be trusted. To setup the application in AAD I followed this Microsoft guide: <a href=\"https:\/\/learn.microsoft.com\/en-us\/azure\/data-factory\/connector-sharepoint-online-list?tabs=data-factory\">Copy data from SharePoint Online List &#8211; Azure Data Factory &amp; Azure Synapse | Microsoft Learn<\/a>.<\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>Search for and select <strong>Azure Active Directory<\/strong> in Azure portal<\/li>\n\n\n\n<li>Inside of AAD under <strong>Manage<\/strong> select <strong>App registrations<\/strong><\/li>\n\n\n\n<li>Click on <strong>New registration<\/strong><\/li>\n\n\n\n<li>Choose a name for your registered app, leave the other values at default and select <strong>Register<\/strong><\/li>\n<\/ol>\n\n\n\n<figure class=\"wp-block-image aligncenter size-large is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"929\" src=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/AppRegistration-1-1024x929.png\" alt=\"\" class=\"wp-image-58\" style=\"width:600px;height:544px\" srcset=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/AppRegistration-1-1024x929.png 1024w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/AppRegistration-1-300x272.png 300w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/AppRegistration-1-768x697.png 768w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/AppRegistration-1.png 1036w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<ol start=\"5\" class=\"wp-block-list\">\n<li>After the registration select your application and under <strong>Manage <\/strong>select <strong>Authentication<\/strong><\/li>\n\n\n\n<li>Under <strong>Platform configurations<\/strong> click on <strong>Add a platform<\/strong><\/li>\n<\/ol>\n\n\n\n<figure class=\"wp-block-image aligncenter size-large is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"861\" src=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/03_AppAddPlattform-1024x861.png\" alt=\"\" class=\"wp-image-60\" style=\"width:600px;height:504px\" srcset=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/03_AppAddPlattform-1024x861.png 1024w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/03_AppAddPlattform-300x252.png 300w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/03_AppAddPlattform-768x646.png 768w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/03_AppAddPlattform.png 1268w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<ol start=\"7\" class=\"wp-block-list\">\n<li>Under <strong>Configure platforms<\/strong> select <strong>Web<\/strong><\/li>\n\n\n\n<li>Enter a redirect URI, a Front-channel logout URL (in our case this does not realy matter) and make sure the checkbox <strong>ID tokens<\/strong> is marked.<\/li>\n<\/ol>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"731\" height=\"830\" src=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/04_AppConfigureWeb.png\" alt=\"\" class=\"wp-image-61\" style=\"width:600px;height:681px\" srcset=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/04_AppConfigureWeb.png 731w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/04_AppConfigureWeb-264x300.png 264w\" sizes=\"auto, (max-width: 731px) 100vw, 731px\" \/><\/figure>\n\n\n\n<ol start=\"9\" class=\"wp-block-list\">\n<li>Under <strong>Manage<\/strong> select <strong>Certificates &amp; secrets<\/strong> and click on <strong>New client secret<\/strong> under <strong>Client secrets (0)<\/strong><\/li>\n\n\n\n<li>Add a Description and a suitable expire date<\/li>\n\n\n\n<li>Make sure you record the secret value for later use, once you leave the page it will never be displayed again!<\/li>\n<\/ol>\n\n\n\n<p class=\"wp-block-paragraph\">We will need the secret value and secret ID as well as the Application (client) ID and Directory (tenant) ID from the <strong>Overview<\/strong> page later for the API calls and to set up permissions in SPO.<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">2. Grant SPO site permission to registered application in AAD<\/h4>\n\n\n\n<p class=\"wp-block-paragraph\">Now that we have created an Identity we need to grant it permission to the SPO site we want to get files from.<\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>Open the SharePoint Online site link: <code>https:\/\/[your_site_url]\/_layouts\/15\/appinv.aspx<\/code>\n<ul class=\"wp-block-list\">\n<li>replace [your_site_url] with the SPO site URL e.g. if your SPO domain name was &#8220;syntera&#8221; and your SPO Site &#8220;blog&#8221; your URL would look like this:\n<ul class=\"wp-block-list\">\n<li><code>https:\/\/<strong>syntera<\/strong>.sharepoint.com\/sites\/<strong>blog<\/strong>\/_layouts\/15\/appinv.aspx<\/code><\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<\/li>\n\n\n\n<li>Fill in the fields as follows:\n<ul class=\"wp-block-list\">\n<li>App Id: Copy the Application (client) ID from your registered app<\/li>\n\n\n\n<li>Title: Choose a title<\/li>\n\n\n\n<li>App Domain: <code>localhost.com<\/code><\/li>\n\n\n\n<li>Redirect URL: <code>https:\/\/www.localhost.com<\/code><\/li>\n\n\n\n<li>Permission Request XML: Insert the following code snippet:<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n\n\n\n<pre class=\"wp-block-code has-tertiary-background-color has-background\" style=\"border-style:none;border-width:0px\"><code>&lt;AppPermissionRequests AllowAppOnlyPolicy=\"true\"&gt;\n    &lt;AppPermissionRequest Scope=\"http:\/\/sharepoint\/content\/sitecollection\/web\" Right=\"Read\"\/&gt;\n&lt;\/AppPermissionRequests&gt;<\/code><\/pre>\n\n\n\n<figure class=\"wp-block-image aligncenter size-large is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"632\" src=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/05_SPOPermission-1024x632.png\" alt=\"\" class=\"wp-image-63\" style=\"width:600px;height:370px\" srcset=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/05_SPOPermission-1024x632.png 1024w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/05_SPOPermission-300x185.png 300w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/05_SPOPermission-768x474.png 768w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/05_SPOPermission.png 1245w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<ol start=\"3\" class=\"wp-block-list\">\n<li>Select <strong>Create<\/strong> and <strong>Trust it<\/strong> on the next page<\/li>\n<\/ol>\n\n\n\n<p class=\"has-background-background-color has-background wp-block-paragraph\"><strong>Important<\/strong>:<br>Azure Access Control (ACS), a service of Azure Active Directory (Azure AD), has been retired on November 7, 2018. This means that if your SPO tenant was created after this date the use of ACS app-only access token is disabled by default. To enable it (necessary for this solution) run <code>set-spotenant -DisableCustomAppAuthentication $false<\/code> from your SharePoint admin PowerShell.<\/p>\n\n\n\n<h4 class=\"wp-block-heading\">3. Provision RessourceGroup, ADF and ADLS<\/h4>\n\n\n\n<ol class=\"wp-block-list\">\n<li>In Azure Portal select <strong>Resource groups<\/strong> under <strong>Navigate<\/strong> and select <strong>Create<\/strong><\/li>\n\n\n\n<li>Choose your subscription, a name (in my case: dev-rg-blog-00000) and a region and select <strong>Review + create<\/strong><\/li>\n\n\n\n<li>In Azure Portal select <strong>Create a resource<\/strong> search for Azure Data Factory and select <strong>Create<\/strong><\/li>\n\n\n\n<li>Choose your subscription, the resource group you created previously, a name (in my case: dev-adf-blog-00000) and a region <\/li>\n\n\n\n<li>Under Git configuration check the box <strong>Configure Git later <\/strong>and select <strong>Review + create<\/strong> (you can configure your Git integration later following another guide if necessary for your project)<\/li>\n\n\n\n<li>In Azure Portal select <strong>Create a resource<\/strong>, search for &#8220;storage&#8221;, select <strong>Storage Account<\/strong> and then <strong>Create<\/strong><\/li>\n\n\n\n<li>Choose your subscription, the resource group you created previously, a name (in my case: devadlsblog00000, a region and change the redundancy to LRS (lowest cost option &#8211; adjust for your case)<\/li>\n\n\n\n<li>In <strong>Advanced<\/strong> under <strong>Data Lake Storage Gen2<\/strong> check the box for Enable hierarchical namespace and select <strong>Review + create<\/strong><\/li>\n\n\n\n<li>Go to your created ADLS resource and under <strong>Data Storage<\/strong> select <strong>Containers<\/strong><\/li>\n\n\n\n<li>Select <strong>+ Container<\/strong> on the top, give the container a name and create it<\/li>\n<\/ol>\n\n\n\n<h4 class=\"wp-block-heading\">4. Create Pipeline in ADF and get access token from SPO<\/h4>\n\n\n\n<p class=\"wp-block-paragraph\">Now that all the prerequisites are set up we can start building the pipeline in ADF. In the first step we are creating a web activity to retrieve an access token from the SPO site using an API call.<\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>Open ADF Studio\n<ul class=\"wp-block-list\">\n<li>In Azure portal under <strong>Navigate<\/strong> select <strong>Resource groups<\/strong> and click on the resource group you created<\/li>\n\n\n\n<li>Inside your resource group you should see the ADF and ADLS resources we created previously<\/li>\n\n\n\n<li>After selecting your ADF resource under <strong>Getting started<\/strong> you should see <strong>Open Azure Data Factory Studio<\/strong>, which will open a new tab<\/li>\n<\/ul>\n<\/li>\n\n\n\n<li>Create a new pipeline\n<ul class=\"wp-block-list\">\n<li>Under <strong>Author<\/strong> right-click <strong>Pipelines<\/strong> and select <strong>New pipeline<\/strong><\/li>\n<\/ul>\n<\/li>\n\n\n\n<li>Create a Web activity\n<ul class=\"wp-block-list\">\n<li>Under <strong>General<\/strong> select <strong>Web<\/strong> and drag it to your pipeline<\/li>\n<\/ul>\n<\/li>\n\n\n\n<li>Under <strong>General<\/strong> choose a suitable name (in my case: GetBearerToken)<\/li>\n\n\n\n<li>In <strong>Settings<\/strong> enter the following information:\n<ul class=\"wp-block-list\">\n<li>URL: <code>https:\/\/<a href=\"https:\/\/accounts.accesscontrol.windows.net\/[Tenant-ID]\/tokens\/OAuth\/2\">accounts.accesscontrol.windows.net\/[Tenant-ID]\/tokens\/OAuth\/2<\/a><\/code>\n<ul class=\"wp-block-list\">\n<li>replace <code>[Tenant-ID]<\/code> with the Directory (tenant) ID from your registered app<\/li>\n<\/ul>\n<\/li>\n\n\n\n<li>Method: POST<\/li>\n\n\n\n<li>Authentication: None<\/li>\n\n\n\n<li>Headers:\n<ul class=\"wp-block-list\">\n<li>Name: Content-Type<\/li>\n\n\n\n<li>Value: <code>application\/x-www-form-urlencoded<\/code><\/li>\n<\/ul>\n<\/li>\n\n\n\n<li>Body: <code>grant_type=client_credentials&amp;client_id=[Client-ID]@[Tenant-ID]&amp;client_secret=[Client-Secret]&amp;resource=00000003-0000-0ff1-ce00-000000000000\/[Tenant-Name].sharepoint.com@[Tenant-ID]<\/code>\n<ul class=\"wp-block-list\">\n<li>replace <code>[Tenant-ID]<\/code> with the Directory (tenant) ID from your registered app<\/li>\n\n\n\n<li>replace <code>[Client-ID]<\/code> with the Application (client) ID from your registered app<\/li>\n\n\n\n<li>replace <code>[Client-Secret]<\/code> with the value of the generated secret from your registered app<\/li>\n\n\n\n<li>replace <code>[Tenant-Name]<\/code> with the SPO domain name (in my case: syntera)<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"955\" height=\"895\" src=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/06_BearerToken.png\" alt=\"\" class=\"wp-image-64\" style=\"width:600px;height:562px\" srcset=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/06_BearerToken.png 955w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/06_BearerToken-300x281.png 300w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/06_BearerToken-768x720.png 768w\" sizes=\"auto, (max-width: 955px) 100vw, 955px\" \/><\/figure>\n\n\n\n<ol start=\"6\" class=\"wp-block-list\">\n<li>Select <strong>Debug<\/strong> to check if everything is set up correctly (check the output of the activity to see the access token)<\/li>\n\n\n\n<li>On <strong>General<\/strong> check the box for Secure output to ensure that your access tokens don&#8217;t get logged in your pipeline runs <\/li>\n<\/ol>\n\n\n\n<h4 class=\"wp-block-heading\">5. Get SPO folder metadata (list of files)<\/h4>\n\n\n\n<p class=\"wp-block-paragraph\">Now that we can retrieve access tokens from SPO we are able to get information\/files from the SPO tenant. In order to copy multiple files from a folder we first have to get a list of all files currently located in that specific folder. To receive this list, we will use an API function from SPO which returns the metadata of a specified folder:<\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>Add another Web activity and connect it with the previous one<\/li>\n\n\n\n<li>Under <strong>General<\/strong> choose a suitable name (in my case: GetSPOFolderMetadata) and check the box for secure input (again to ensure the access token does not get logged in your pipeline runs)<\/li>\n\n\n\n<li>Under <strong>Settings<\/strong> put in the following information:\n<ul class=\"wp-block-list\">\n<li>URL: <code>https:\/\/[sharepoint-domain-name].sharepoint.com\/sites\/[sharepoint-site]\/_api\/web\/GetFolderByServerRelativeUrl('\/sites\/[sharepoint-site]\/[relative-path-to-folder]')\/Files<\/code>\n<ul class=\"wp-block-list\">\n<li>replace <code>[sharepoint-domain-name]<\/code> with your SPO domain name (in my case: syntera)<\/li>\n\n\n\n<li>replace <code><code>[sharepoint-site]<\/code><\/code> with your SPO site name (in my case: blog)<\/li>\n\n\n\n<li>replace <code><code><code>[relative-path-to-file]<\/code><\/code><\/code> with the relative URL of your folder (you can check the folder path in SPO by right-clicking on the folder, selecting <strong>Details<\/strong> and selecting <strong>More details<\/strong> in the bottom of the window on the right)<\/li>\n<\/ul>\n<\/li>\n\n\n\n<li>Method: GET<\/li>\n\n\n\n<li>Authentication: None<\/li>\n\n\n\n<li>Headers (there are two headers):\n<ul class=\"wp-block-list\">\n<li>Name1: Authorization<\/li>\n\n\n\n<li>Value1: <code>@{concat('Bearer ', activity('GetBearerToken').output.access_token)}<\/code><\/li>\n\n\n\n<li>Replace GetBearerToken with the name you gave to the first web activity<\/li>\n\n\n\n<li>Name2: Accept<\/li>\n\n\n\n<li>Value2: <code>application\/json<\/code><\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"996\" height=\"876\" src=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/07_SPOMetadata.png\" alt=\"\" class=\"wp-image-66\" style=\"width:600px;height:527px\" srcset=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/07_SPOMetadata.png 996w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/07_SPOMetadata-300x264.png 300w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/07_SPOMetadata-768x675.png 768w\" sizes=\"auto, (max-width: 996px) 100vw, 996px\" \/><\/figure>\n\n\n\n<ol start=\"4\" class=\"wp-block-list\">\n<li>Select <strong>Debug<\/strong> to check if everything is set up correctly (check the output of the activity to see the folder metadata)<\/li>\n<\/ol>\n\n\n\n<h4 class=\"wp-block-heading\">6. Iterate through list of files and copy each file to ADLS<\/h4>\n\n\n\n<p class=\"wp-block-paragraph\">With the list of files, we received as an output of the last web activity, we can now setup a copy job, which copies the files to ADLS. For this task we need a ForEach activity to loop trough every item in the list and a copy activity to actually copy the data. For the copy activity we need to define a source and a sink dataset, which themselves refer to linked services that define the place where a sink or source is located. In our case the sink will be the ADLS, which has a supported connector, the linked service for the source however will be managed through another API call.<\/p>\n\n\n\n<ol class=\"wp-block-list\">\n<li>Add a ForEach activity and connect it with the previous web activity<\/li>\n\n\n\n<li>Under <strong>Settings<\/strong> check the box Sequential and in Items enter the following: <code>@activity('GetSPOFolderMetadata').output.value<\/code>\n<ul class=\"wp-block-list\">\n<li>Replace GetSPOFolderMetadata with the name you chose for the second web activity<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"979\" height=\"645\" src=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/08_ForEach.png\" alt=\"\" class=\"wp-image-68\" style=\"width:600px;height:395px\" srcset=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/08_ForEach.png 979w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/08_ForEach-300x198.png 300w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/08_ForEach-768x506.png 768w\" sizes=\"auto, (max-width: 979px) 100vw, 979px\" \/><\/figure>\n\n\n\n<ol start=\"3\" class=\"wp-block-list\">\n<li>Inside the ForEach activity add a copy data activity and rename it (in my case: Copy data from SPO to ADLS)<\/li>\n\n\n\n<li>Create source linked service<ul><li>In <strong>Manage<\/strong> under <strong>Connections<\/strong> select <strong>Linked services<\/strong> and click on <strong>New<\/strong><\/li><li>In <strong>New linked service<\/strong> search for &#8220;HTTP&#8221; and select it.<\/li><li>Define a name for your linked service (in my case: ls_spo_Blog)<\/li><\/ul>\n<ul class=\"wp-block-list\">\n<li>Set Authentication type to Anonymous<\/li>\n\n\n\n<li>Scroll down to <strong>Parameters<\/strong> select <strong>New<\/strong> and call it RelativeURL<\/li>\n\n\n\n<li>For the Base URL enter the following URL: <code>https:\/\/[sharepoint-domain-name].sharepoint.com\/sites\/[sharepoint-site]\/_api\/web\/GetFileByServerRelativeUrl('@{linkedService().RelativeURL}')\/$value<\/code>\n<ul class=\"wp-block-list\">\n<li>replace <code>[sharepoint-domain-name]<\/code> with your SPO domain name (in my case: syntera)<\/li>\n\n\n\n<li>replace <code><code>[sharepoint-site]<\/code><\/code> with your SPO site name (in my case: blog)<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n\n\n\n<figure class=\"wp-block-image aligncenter size-large is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"710\" height=\"1024\" src=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/09_LSSPO-2-710x1024.png\" alt=\"\" class=\"wp-image-72\" style=\"width:600px;height:865px\" srcset=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/09_LSSPO-2-710x1024.png 710w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/09_LSSPO-2-208x300.png 208w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/09_LSSPO-2-768x1107.png 768w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/09_LSSPO-2.png 788w\" sizes=\"auto, (max-width: 710px) 100vw, 710px\" \/><\/figure>\n\n\n\n<ol start=\"5\" class=\"wp-block-list\">\n<li>Create source dataset<ul><li>Right-click on <strong>Datasets<\/strong> and select <strong>New dataset<\/strong><\/li><li>Under <strong>New dataset<\/strong> search for &#8220;HTTP&#8221; and select it<\/li><li>Under <strong>Select format<\/strong> choose the suitable format for your data (in my case the files are .pdf so I chose binary)<\/li><li>In <strong>Set properties<\/strong> define a name (in my case: ds_spo_Blog) and select the linked service you created previously<\/li><\/ul>\n<ul class=\"wp-block-list\">\n<li>In the overview of the dataset under <strong>Connection<\/strong> in <strong>Linked service properties<\/strong> enter: <code>@dataset().RelativeUR<\/code>L<\/li>\n\n\n\n<li>Under <strong>Parameter<\/strong> select <strong>New<\/strong> and Call it RelativeURL<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n\n\n\n<figure class=\"wp-block-image aligncenter size-large is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"1024\" height=\"649\" src=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/10_DSSPO-1024x649.png\" alt=\"\" class=\"wp-image-73\" style=\"width:600px;height:380px\" srcset=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/10_DSSPO-1024x649.png 1024w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/10_DSSPO-300x190.png 300w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/10_DSSPO-768x487.png 768w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/10_DSSPO.png 1152w\" sizes=\"auto, (max-width: 1024px) 100vw, 1024px\" \/><\/figure>\n\n\n\n<ol start=\"6\" class=\"wp-block-list\">\n<li>Setup parameters for copy data activity\n<ul class=\"wp-block-list\">\n<li>Go back to the copy data activity<\/li>\n\n\n\n<li>Under <strong>Source<\/strong> in <strong>Dataset properties<\/strong> enter: <code>@{item().ServerRelativeUrl}<\/code><\/li>\n\n\n\n<li>Request method: GET<\/li>\n\n\n\n<li>Additional headers: @{concat(&#8216;Authorization: Bearer &#8216;, activity(&#8216;GetBearerToken&#8217;).output.access_token)}\n<ul class=\"wp-block-list\">\n<li>Replace GetBearerToken with the name you gave to the first web activity<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"704\" height=\"689\" src=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/11_copydatasource-1.png\" alt=\"\" class=\"wp-image-81\" style=\"width:600px;height:587px\" srcset=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/11_copydatasource-1.png 704w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/11_copydatasource-1-300x294.png 300w\" sizes=\"auto, (max-width: 704px) 100vw, 704px\" \/><\/figure>\n\n\n\n<ol start=\"7\" class=\"wp-block-list\">\n<li>Create sink dataset and linked service:<ul><li>Under <strong>Sink <\/strong>select <strong>New<\/strong> to create a new source dataset<\/li><\/ul><ul><li>Under <strong>New dataset<\/strong> search for choose <strong>Azure Data Lake Storage Gen2<\/strong> and continue<\/li><li>Under <strong>Select format<\/strong> choose the suitable format for your data (in my case the files are .pdf so I chose binary)<\/li><li>In <strong>Set properties<\/strong> define a name (in my case: ds_adls_Blog)<\/li><\/ul>\n<ul class=\"wp-block-list\">\n<li>For <strong>Linked service <\/strong>select <strong>New<\/strong><\/li>\n\n\n\n<li>Define a name for your linked service (in my case: ls_adls_Blog)<\/li>\n\n\n\n<li>Under <strong>Account selection method<\/strong> select your subscription and the storage account name from the drop down<\/li>\n<\/ul>\n<\/li>\n<\/ol>\n\n\n\n<figure class=\"wp-block-image aligncenter size-large is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"670\" height=\"1024\" src=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/12_LSadls-1-670x1024.png\" alt=\"\" class=\"wp-image-77\" style=\"width:600px;height:917px\" srcset=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/12_LSadls-1-670x1024.png 670w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/12_LSadls-1-196x300.png 196w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/12_LSadls-1-768x1174.png 768w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/12_LSadls-1.png 786w\" sizes=\"auto, (max-width: 670px) 100vw, 670px\" \/><\/figure>\n\n\n\n<ol start=\"8\" class=\"wp-block-list\">\n<li>Back in <strong>Set properties<\/strong> select the <strong>Linked service<\/strong> you just created and in the file path enter the container name<\/li>\n<\/ol>\n\n\n\n<figure class=\"wp-block-image aligncenter size-full is-resized\"><img loading=\"lazy\" decoding=\"async\" width=\"784\" height=\"506\" src=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/13_DSadls.png\" alt=\"\" class=\"wp-image-78\" style=\"width:600px;height:387px\" srcset=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/13_DSadls.png 784w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/13_DSadls-300x194.png 300w, https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/13_DSadls-768x496.png 768w\" sizes=\"auto, (max-width: 784px) 100vw, 784px\" \/><\/figure>\n\n\n\n<ol start=\"9\" class=\"wp-block-list\">\n<li>Select <strong>Debug<\/strong> to check if everything is set up correctly, if successful check your blob storage to validate that the file was copied.<\/li>\n<\/ol>\n\n\n\n<h4 class=\"wp-block-heading\">Further Considerations<\/h4>\n\n\n\n<p class=\"wp-block-paragraph\">After all these steps we now have a working solution but there are some further things to be considered. We secured the output of the &#8216;GetBearerToken&#8217; activity and the input of the &#8216;GetSPOFolderMetadata&#8217; activity to ensure the access token does not get logged. However, the secret of the app we registered is still hard coded in the &#8216;GetBearerToken&#8217; activity, which means it will get logged in your pipeline runs. Consider setting up an Azure Key Vault to secure the secret of the registered app. For reusability purposes in your company, I would also advise you to parametrize all the inputs to the pipeline (e.g. the SPO Site, the application and client ID etc.). It is a onetime effort which will make your life way easier in the future when applying this pipeline on a different use case.<\/p>\n","protected":false},"excerpt":{"rendered":"<p>Azure Data Factory is a great tool for ETL pipelines, and we love working with it. However, when it comes to the integration with the rest of the non-Azure Microsoft world (especially SharePoint) it can get a bit frustrating. In a recent project I wanted to build a solution, which allows employees to upload documents [&hellip;]<\/p>\n","protected":false},"author":2,"featured_media":82,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[7],"tags":[12,13],"class_list":["post-55","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-data-engineering","tag-azure-data-factory","tag-sharepoint"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v21.8 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>Copy files from SharePoint to Blob Storage using Azure Data Factory - Syntera<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.syntera.ch\/blog\/2022\/10\/10\/copy-files-from-sharepoint-to-blob-storage-using-azure-data-factory\/\" \/>\n<meta property=\"og:locale\" content=\"en_US\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"Copy files from SharePoint to Blob Storage using Azure Data Factory - Syntera\" \/>\n<meta property=\"og:description\" content=\"Azure Data Factory is a great tool for ETL pipelines, and we love working with it. However, when it comes to the integration with the rest of the non-Azure Microsoft world (especially SharePoint) it can get a bit frustrating. In a recent project I wanted to build a solution, which allows employees to upload documents [&hellip;]\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.syntera.ch\/blog\/2022\/10\/10\/copy-files-from-sharepoint-to-blob-storage-using-azure-data-factory\/\" \/>\n<meta property=\"og:site_name\" content=\"Syntera\" \/>\n<meta property=\"article:published_time\" content=\"2022-10-10T13:37:34+00:00\" \/>\n<meta property=\"article:modified_time\" content=\"2024-07-18T13:38:57+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/14_architecture.png\" \/>\n\t<meta property=\"og:image:width\" content=\"985\" \/>\n\t<meta property=\"og:image:height\" content=\"330\" \/>\n\t<meta property=\"og:image:type\" content=\"image\/png\" \/>\n<meta name=\"author\" content=\"Dimitri B\u00fctikofer\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"Written by\" \/>\n\t<meta name=\"twitter:data1\" content=\"Dimitri B\u00fctikofer\" \/>\n\t<meta name=\"twitter:label2\" content=\"Est. reading time\" \/>\n\t<meta name=\"twitter:data2\" content=\"12 minutes\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.syntera.ch\/blog\/2022\/10\/10\/copy-files-from-sharepoint-to-blob-storage-using-azure-data-factory\/\",\"url\":\"https:\/\/www.syntera.ch\/blog\/2022\/10\/10\/copy-files-from-sharepoint-to-blob-storage-using-azure-data-factory\/\",\"name\":\"Copy files from SharePoint to Blob Storage using Azure Data Factory - Syntera\",\"isPartOf\":{\"@id\":\"https:\/\/www.syntera.ch\/blog\/#website\"},\"datePublished\":\"2022-10-10T13:37:34+00:00\",\"dateModified\":\"2024-07-18T13:38:57+00:00\",\"author\":{\"@id\":\"https:\/\/www.syntera.ch\/blog\/#\/schema\/person\/843f812ce16de687e4d4527aee313034\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.syntera.ch\/blog\/2022\/10\/10\/copy-files-from-sharepoint-to-blob-storage-using-azure-data-factory\/#breadcrumb\"},\"inLanguage\":\"en-US\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.syntera.ch\/blog\/2022\/10\/10\/copy-files-from-sharepoint-to-blob-storage-using-azure-data-factory\/\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.syntera.ch\/blog\/2022\/10\/10\/copy-files-from-sharepoint-to-blob-storage-using-azure-data-factory\/#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"Home\",\"item\":\"https:\/\/www.syntera.ch\/blog\/\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"Copy files from SharePoint to Blob Storage using Azure Data Factory\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.syntera.ch\/blog\/#website\",\"url\":\"https:\/\/www.syntera.ch\/blog\/\",\"name\":\"Syntera\",\"description\":\"translating data into business value.\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.syntera.ch\/blog\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"en-US\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.syntera.ch\/blog\/#\/schema\/person\/843f812ce16de687e4d4527aee313034\",\"name\":\"Dimitri B\u00fctikofer\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"en-US\",\"@id\":\"https:\/\/www.syntera.ch\/blog\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/secure.gravatar.com\/avatar\/9cbefca9184e6b8146d83f72900473b791b31bc6c2a10a8e57f77e9c41f11b45?s=96&d=mm&r=g\",\"contentUrl\":\"https:\/\/secure.gravatar.com\/avatar\/9cbefca9184e6b8146d83f72900473b791b31bc6c2a10a8e57f77e9c41f11b45?s=96&d=mm&r=g\",\"caption\":\"Dimitri B\u00fctikofer\"},\"sameAs\":[\"http:\/\/www.syntera.ch\"],\"url\":\"https:\/\/www.syntera.ch\/blog\/author\/dimitri\/\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"Copy files from SharePoint to Blob Storage using Azure Data Factory - Syntera","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.syntera.ch\/blog\/2022\/10\/10\/copy-files-from-sharepoint-to-blob-storage-using-azure-data-factory\/","og_locale":"en_US","og_type":"article","og_title":"Copy files from SharePoint to Blob Storage using Azure Data Factory - Syntera","og_description":"Azure Data Factory is a great tool for ETL pipelines, and we love working with it. However, when it comes to the integration with the rest of the non-Azure Microsoft world (especially SharePoint) it can get a bit frustrating. In a recent project I wanted to build a solution, which allows employees to upload documents [&hellip;]","og_url":"https:\/\/www.syntera.ch\/blog\/2022\/10\/10\/copy-files-from-sharepoint-to-blob-storage-using-azure-data-factory\/","og_site_name":"Syntera","article_published_time":"2022-10-10T13:37:34+00:00","article_modified_time":"2024-07-18T13:38:57+00:00","og_image":[{"width":985,"height":330,"url":"https:\/\/www.syntera.ch\/blog\/wp-content\/uploads\/2022\/10\/14_architecture.png","type":"image\/png"}],"author":"Dimitri B\u00fctikofer","twitter_card":"summary_large_image","twitter_misc":{"Written by":"Dimitri B\u00fctikofer","Est. reading time":"12 minutes"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.syntera.ch\/blog\/2022\/10\/10\/copy-files-from-sharepoint-to-blob-storage-using-azure-data-factory\/","url":"https:\/\/www.syntera.ch\/blog\/2022\/10\/10\/copy-files-from-sharepoint-to-blob-storage-using-azure-data-factory\/","name":"Copy files from SharePoint to Blob Storage using Azure Data Factory - Syntera","isPartOf":{"@id":"https:\/\/www.syntera.ch\/blog\/#website"},"datePublished":"2022-10-10T13:37:34+00:00","dateModified":"2024-07-18T13:38:57+00:00","author":{"@id":"https:\/\/www.syntera.ch\/blog\/#\/schema\/person\/843f812ce16de687e4d4527aee313034"},"breadcrumb":{"@id":"https:\/\/www.syntera.ch\/blog\/2022\/10\/10\/copy-files-from-sharepoint-to-blob-storage-using-azure-data-factory\/#breadcrumb"},"inLanguage":"en-US","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.syntera.ch\/blog\/2022\/10\/10\/copy-files-from-sharepoint-to-blob-storage-using-azure-data-factory\/"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.syntera.ch\/blog\/2022\/10\/10\/copy-files-from-sharepoint-to-blob-storage-using-azure-data-factory\/#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"Home","item":"https:\/\/www.syntera.ch\/blog\/"},{"@type":"ListItem","position":2,"name":"Copy files from SharePoint to Blob Storage using Azure Data Factory"}]},{"@type":"WebSite","@id":"https:\/\/www.syntera.ch\/blog\/#website","url":"https:\/\/www.syntera.ch\/blog\/","name":"Syntera","description":"translating data into business value.","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.syntera.ch\/blog\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"en-US"},{"@type":"Person","@id":"https:\/\/www.syntera.ch\/blog\/#\/schema\/person\/843f812ce16de687e4d4527aee313034","name":"Dimitri B\u00fctikofer","image":{"@type":"ImageObject","inLanguage":"en-US","@id":"https:\/\/www.syntera.ch\/blog\/#\/schema\/person\/image\/","url":"https:\/\/secure.gravatar.com\/avatar\/9cbefca9184e6b8146d83f72900473b791b31bc6c2a10a8e57f77e9c41f11b45?s=96&d=mm&r=g","contentUrl":"https:\/\/secure.gravatar.com\/avatar\/9cbefca9184e6b8146d83f72900473b791b31bc6c2a10a8e57f77e9c41f11b45?s=96&d=mm&r=g","caption":"Dimitri B\u00fctikofer"},"sameAs":["http:\/\/www.syntera.ch"],"url":"https:\/\/www.syntera.ch\/blog\/author\/dimitri\/"}]}},"_links":{"self":[{"href":"https:\/\/www.syntera.ch\/blog\/wp-json\/wp\/v2\/posts\/55","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.syntera.ch\/blog\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.syntera.ch\/blog\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.syntera.ch\/blog\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.syntera.ch\/blog\/wp-json\/wp\/v2\/comments?post=55"}],"version-history":[{"count":21,"href":"https:\/\/www.syntera.ch\/blog\/wp-json\/wp\/v2\/posts\/55\/revisions"}],"predecessor-version":[{"id":714,"href":"https:\/\/www.syntera.ch\/blog\/wp-json\/wp\/v2\/posts\/55\/revisions\/714"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/www.syntera.ch\/blog\/wp-json\/wp\/v2\/media\/82"}],"wp:attachment":[{"href":"https:\/\/www.syntera.ch\/blog\/wp-json\/wp\/v2\/media?parent=55"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.syntera.ch\/blog\/wp-json\/wp\/v2\/categories?post=55"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.syntera.ch\/blog\/wp-json\/wp\/v2\/tags?post=55"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}