1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
#![allow(missing_docs)]

use std::vec::Vec;
use std::io::{Read, Write};
use std::fs::File;
use std::path::{Path, PathBuf};

#[cfg(feature = "upgrade")]
use semver::Version;

use serde_json;
use sha1;
use hyper::{self, Client};
use hyper::net::HttpsConnector;
use hyper::header::{Authorization, Basic};
use hyper::status::StatusCode;
use hyper_native_tls::NativeTlsClient;

use core::{CliError, LalResult};


/// Artifactory credentials
#[derive(Serialize, Deserialize, Clone)]
pub struct Credentials {
    /// Upload username
    pub username: String,
    /// Upload password
    pub password: String,
}

/// Static Artifactory locations
#[derive(Serialize, Deserialize, Clone, Default)]
pub struct ArtifactoryConfig {
    /// Location of artifactory API master (for API queries)
    pub master: String,
    /// Location of artifactory slave (for fetching artifacts)
    pub slave: String,
    /// Release group name (for API queries)
    pub release: String,
    /// Virtual group (for downloads)
    pub vgroup: String,
    /// Optional publish credentials
    pub credentials: Option<Credentials>,
}


// Need these to query for stored artifacts:
// This query has tons of info, but we only care about the version
// And the version is encoded in children.uri with leading slash
#[derive(Deserialize)]
struct ArtifactoryVersion {
    uri: String, // folder: bool,
}
#[derive(Deserialize)]
struct ArtifactoryStorageResponse {
    children: Vec<ArtifactoryVersion>,
}

// simple request body fetcher
fn hyper_req(url: &str) -> LalResult<String> {
    let client = Client::with_connector(HttpsConnector::new(NativeTlsClient::new().unwrap()));
    let mut res = client.get(url).send()?;
    if res.status != hyper::Ok {
        return Err(CliError::BackendFailure(format!("GET request with {}", res.status)));
    }
    let mut body = String::new();
    res.read_to_string(&mut body)?;
    Ok(body)
}

// simple request downloader
pub fn http_download_to_path(url: &str, save: &PathBuf) -> LalResult<()> {
    debug!("GET {}", url);
    let client = Client::with_connector(HttpsConnector::new(NativeTlsClient::new().unwrap()));
    let mut res = client.get(url).send()?;
    if res.status != hyper::Ok {
        return Err(CliError::BackendFailure(format!("GET request with {}", res.status)));
    }

    if cfg!(feature = "progress") {
        #[cfg(feature = "progress")]
        {
            use indicatif::{ProgressBar, ProgressStyle};
            let total_size = res.headers.get::<hyper::header::ContentLength>().unwrap().0;
            let mut downloaded = 0;
            let mut buffer = [0; 1024 * 64];
            let mut f = File::create(save)?;
            let pb = ProgressBar::new(total_size);
            pb.set_style(ProgressStyle::default_bar()
                             .template("{bar:40.yellow/black} {bytes}/{total_bytes} ({eta})"));

            while downloaded < total_size {
                let read = res.read(&mut buffer)?;
                f.write_all(&buffer[0..read])?;
                downloaded += read as u64;
                pb.set_position(downloaded);
            }
            f.flush()?;
        }
    } else {
        let mut buffer: Vec<u8> = Vec::new();
        res.read_to_end(&mut buffer)?;
        let mut f = File::create(save)?;
        f.write_all(&buffer)?;
    }
    Ok(())
}


/// Query the Artifactory storage api
///
/// This will get, then parse all results as u32s, and return this list.
/// This assumes versoning is done via a single integer.
fn get_storage_versions(uri: &str) -> LalResult<Vec<u32>> {
    debug!("GET {}", uri);

    let resp = hyper_req(uri)
        .map_err(|e| {
            warn!("Failed to GET {}: {}", uri, e);
            CliError::BackendFailure("No version information found on API".into())
        })?;

    trace!("Got body {}", resp);

    let res: ArtifactoryStorageResponse = serde_json::from_str(&resp)?;
    let builds: Vec<u32> = res.children
        .iter()
        .map(|r| r.uri.as_str())
        .map(|r| r.trim_matches('/'))
        .filter_map(|b| b.parse().ok())
        .collect();
    Ok(builds)
}

// artifactory extra headers
header! {(XCheckSumDeploy, "X-Checksum-Deploy") => [String]}
header! {(XCheckSumSha1, "X-Checksum-Sha1") => [String]}

/// Upload a tarball to artifactory
///
/// This is using a http basic auth PUT to artifactory using config credentials.
fn upload_artifact(arti: &ArtifactoryConfig, uri: &str, f: &mut File) -> LalResult<()> {
    if let Some(creds) = arti.credentials.clone() {
        let client = Client::new();

        let mut buffer: Vec<u8> = Vec::new();
        f.read_to_end(&mut buffer)?;

        let full_uri = format!("{}/{}/{}", arti.slave, arti.release, uri);

        let mut sha = sha1::Sha1::new();
        sha.update(&buffer);

        let auth = Authorization(Basic {
                                     username: creds.username,
                                     password: Some(creds.password),
                                 });

        // upload the artifact
        info!("PUT {}", full_uri);
        let resp = client.put(&full_uri[..]).header(auth.clone()).body(&buffer[..]).send()?;
        debug!("resp={:?}", resp);
        let respstr = format!("{} from PUT {}", resp.status, full_uri);
        if resp.status != StatusCode::Created {
            return Err(CliError::UploadFailure(respstr));
        }
        debug!("{}", respstr);

        // do another request to get the hash on artifactory
        // jfrog api does not allow do do both at once - and this also creates the md5 (somehow)
        // this creates ${full_uri}.sha1 and ${full_uri}.md5 (although we just gave it the sha..)
        // This `respsha` can fail if engci-maven becomes inconsistent. NotFound has been seen.
        // And that makes no sense because the above must have returned Created to get here..
        info!("PUT {} (X-Checksum-Sha1)", full_uri);
        let respsha = client
            .put(&full_uri[..])
            .header(XCheckSumDeploy("true".into()))
            .header(XCheckSumSha1(sha.digest().to_string()))
            .header(auth)
            .send()?;
        debug!("respsha={:?}", respsha);
        let respshastr = format!("{} from PUT {} (X-Checksum-Sha1)", respsha.status, full_uri);
        if respsha.status != StatusCode::Created {
            return Err(CliError::UploadFailure(respshastr));
        }
        debug!("{}", respshastr);

        Ok(())
    } else {
        Err(CliError::MissingBackendCredentials)
    }
}

/// Get the maximal version number from the storage api
fn get_storage_as_u32(uri: &str) -> LalResult<u32> {
    if let Some(&latest) = get_storage_versions(uri)?.iter().max() {
        Ok(latest)
    } else {
        Err(CliError::BackendFailure("No version information found on API".into()))
    }
}

// The URL for a component tarball stored in the default artifactory location
fn get_dependency_url_default(art_cfg: &ArtifactoryConfig, name: &str, version: u32) -> String {
    let tar_url = format!("{}/{}/{}/{}/{}.tar.gz",
                          art_cfg.slave,
                          art_cfg.vgroup,
                          name,
                          version.to_string(),
                          name);

    trace!("Inferring tarball location as {}", tar_url);
    tar_url
}

// The URL for a component tarball under the one of the environment trees
fn get_dependency_env_url(
    art_cfg: &ArtifactoryConfig,
    name: &str,
    version: u32,
    env: &str,
) -> String {
    let tar_url = format!("{}/{}/env/{}/{}/{}/{}.tar.gz",
                          art_cfg.slave,
                          art_cfg.vgroup,
                          env,
                          name,
                          version.to_string(),
                          name);

    trace!("Inferring tarball location as {}", tar_url);
    tar_url
}

fn get_dependency_url(
    art_cfg: &ArtifactoryConfig,
    name: &str,
    version: u32,
    env: Option<&str>,
) -> String {
    if let Some(e) = env {
        get_dependency_env_url(art_cfg, name, version, e)
    } else {
        // This is only used by lal export without -e
        get_dependency_url_default(art_cfg, name, version)
    }
}

fn get_dependency_url_latest(
    art_cfg: &ArtifactoryConfig,
    name: &str,
    env: Option<&str>,
) -> LalResult<Component> {
    let url = format!("{}/api/storage/{}/{}",
                      art_cfg.master,
                      art_cfg.release,
                      name);
    let v = get_storage_as_u32(&url)?;

    debug!("Found latest version as {}", v);
    Ok(Component {
           tarball: get_dependency_url(art_cfg, name, v, env),
           version: v,
           name: name.into(),
       })
}

// This queries the API for the default location
// if a default exists, then all our current multi-builds must exist
fn get_latest_versions(
    art_cfg: &ArtifactoryConfig,
    name: &str,
    env: Option<&str>,
) -> LalResult<Vec<u32>> {
    let url = match env {
        Some(e) => {
            format!("{}/api/storage/{}/{}/{}/{}",
                    art_cfg.master,
                    art_cfg.release,
                    "env",
                    e,
                    name)
        }
        None => {
            format!("{}/api/storage/{}/{}",
                    art_cfg.master,
                    art_cfg.release,
                    name)
        }
    };
    get_storage_versions(&url)
}

/// Main entry point for install
fn get_tarball_uri(
    art_cfg: &ArtifactoryConfig,
    name: &str,
    version: Option<u32>,
    env: Option<&str>,
) -> LalResult<Component> {
    if let Some(v) = version {
        Ok(Component {
               tarball: get_dependency_url(art_cfg, name, v, env),
               version: v,
               name: name.into(),
           })
    } else {
        get_dependency_url_latest(art_cfg, name, env)
    }
}

/// Latest lal version - as seen on artifactory
#[cfg(feature = "upgrade")]
pub struct LatestLal {
    /// URL of the latest tarball
    pub url: String,
    /// Semver::Version of the latest tarball
    pub version: Version,
}

/// Entry point for `lal::upgrade`
///
/// This mostly duplicates the behaviour in `get_storage_as_u32`, however,
/// it is parsing the version as a `semver::Version` struct rather than a u32.
/// This is used regardless of your used backend because we want people to use our
/// main release of lal on CME-release on cisco artifactory at the moment.
#[cfg(feature = "upgrade")]
pub fn get_latest_lal_version() -> LalResult<LatestLal> {
    // canonical latest url
    let uri = "https://engci-maven-master.cisco.com/artifactory/api/storage/CME-release/lal";
    debug!("GET {}", uri);
    let resp = hyper_req(uri)
        .map_err(|e| {
            warn!("Failed to GET {}: {}", uri, e);
            CliError::BackendFailure("No version information found on API".into())
        })?;
    trace!("Got body {}", resp);

    let res: ArtifactoryStorageResponse = serde_json::from_str(&resp)?;
    let latest: Option<Version> = res.children
        .iter()
        .map(|r| r.uri.trim_matches('/').to_string())
        .inspect(|v| trace!("Found lal version {}", v))
        .filter_map(|v| Version::parse(&v).ok())
        .max(); // Semver::Version implements an order

    if let Some(l) = latest {
        Ok(LatestLal {
               version: l.clone(),
               url: format!("https://engci-maven.cisco.com/artifactory/CME-group/lal/{}/lal.tar",
                            l),
           })
    } else {
        warn!("Failed to parse version information from artifactory storage api for lal");
        Err(CliError::BackendFailure("No version information found on API".into()))
    }
}

use super::{Backend, Component};

/// Everything we need for Artifactory to implement the Backend trait
pub struct ArtifactoryBackend {
    /// Artifactory config and credentials
    pub config: ArtifactoryConfig,
    /// Cache directory
    pub cache: String,
}

impl ArtifactoryBackend {
    pub fn new(cfg: &ArtifactoryConfig, cache: &str) -> Self {
        // TODO: create hyper clients in here rather than once per download
        ArtifactoryBackend {
            config: cfg.clone(),
            cache: cache.into(),
        }
    }
}

/// Artifact backend trait for `ArtifactoryBackend`
///
/// This is intended to be used by the caching trait `CachedBackend`, but for
/// specific low-level use cases, these methods can be used directly.
impl Backend for ArtifactoryBackend {
    fn get_versions(&self, name: &str, loc: Option<&str>) -> LalResult<Vec<u32>> {
        get_latest_versions(&self.config, name, loc)
    }

    fn get_latest_version(&self, name: &str, loc: Option<&str>) -> LalResult<u32> {
        let latest = get_dependency_url_latest(&self.config, name, loc)?;
        Ok(latest.version)
    }

    fn get_tarball_url(
        &self,
        name: &str,
        version: Option<u32>,
        loc: Option<&str>,
    ) -> LalResult<Component> {
        get_tarball_uri(&self.config, name, version, loc)
    }

    fn upload_artifact_dir(&self, name: &str, version: u32, env: Option<&str>) -> LalResult<()> {
        // this fn basically assumes all the sanity checks have been performed
        // files must exist and lockfile must be sensible
        let artdir = Path::new("./ARTIFACT");
        let tarball = artdir.join(format!("{}.tar.gz", name));
        let lockfile = artdir.join("lockfile.json");

        // uri prefix if specific env upload
        let prefix = env.map(|s| format!("env/{}/", s)).unwrap_or_else(|| "".into());

        let tar_uri = format!("{}{}/{}/{}.tar.gz", prefix, name, version, name);
        let mut tarf = File::open(tarball)?;
        upload_artifact(&self.config, &tar_uri, &mut tarf)?;

        let mut lockf = File::open(lockfile)?;
        let lf_uri = format!("{}{}/{}/lockfile.json", prefix, name, version);
        upload_artifact(&self.config, &lf_uri, &mut lockf)?;
        Ok(())
    }

    fn get_cache_dir(&self) -> String { self.cache.clone() }

    fn raw_download(&self, url: &str, dest: &PathBuf) -> LalResult<()> {
        http_download_to_path(url, dest)
    }
}